diff --git a/.github/workflows/fix-flakes.yml b/.github/workflows/fix-flakes.yml index 740a857bdf70e..346f7359378b4 100644 --- a/.github/workflows/fix-flakes.yml +++ b/.github/workflows/fix-flakes.yml @@ -272,7 +272,7 @@ jobs: - uses: actions/create-github-app-token@v3 id: app-token with: - app-id: ${{ vars.PLAYWRIGHT_APP_ID }} + client-id: ${{ vars.PLAYWRIGHT_APP_CLIENT_ID }} private-key: ${{ secrets.PLAYWRIGHT_PRIVATE_KEY }} - name: Apply commit and open PR diff --git a/.github/workflows/publish_release.yml b/.github/workflows/publish_release.yml index 354e0d98e954e..18c678511d196 100644 --- a/.github/workflows/publish_release.yml +++ b/.github/workflows/publish_release.yml @@ -58,7 +58,7 @@ jobs: - uses: actions/create-github-app-token@v3 id: app-token with: - app-id: ${{ vars.PLAYWRIGHT_APP_ID }} + client-id: ${{ vars.PLAYWRIGHT_APP_CLIENT_ID }} private-key: ${{ secrets.PLAYWRIGHT_PRIVATE_KEY }} repositories: trace.playwright.dev - name: Deploy Canary diff --git a/.github/workflows/roll_browser_into_playwright.yml b/.github/workflows/roll_browser_into_playwright.yml index 93430568f8b74..007eef766c197 100644 --- a/.github/workflows/roll_browser_into_playwright.yml +++ b/.github/workflows/roll_browser_into_playwright.yml @@ -55,7 +55,7 @@ jobs: - uses: actions/create-github-app-token@v3 id: app-token with: - app-id: ${{ vars.PLAYWRIGHT_APP_ID }} + client-id: ${{ vars.PLAYWRIGHT_APP_CLIENT_ID }} private-key: ${{ secrets.PLAYWRIGHT_PRIVATE_KEY }} - name: Create Pull Request uses: actions/github-script@v9 diff --git a/.github/workflows/roll_nodejs.yml b/.github/workflows/roll_nodejs.yml index 61777ac1ec33a..1287bbe7c9c0a 100644 --- a/.github/workflows/roll_nodejs.yml +++ b/.github/workflows/roll_nodejs.yml @@ -36,7 +36,7 @@ jobs: - uses: actions/create-github-app-token@v3 id: app-token with: - app-id: ${{ vars.PLAYWRIGHT_APP_ID }} + client-id: ${{ vars.PLAYWRIGHT_APP_CLIENT_ID }} private-key: ${{ secrets.PLAYWRIGHT_PRIVATE_KEY }} - name: Create Pull Request if: ${{ steps.prepare-branch.outputs.HAS_CHANGES == '1' }} diff --git a/.github/workflows/roll_stable_test_runner.yml b/.github/workflows/roll_stable_test_runner.yml index af229b11dce72..0e262839a61a0 100644 --- a/.github/workflows/roll_stable_test_runner.yml +++ b/.github/workflows/roll_stable_test_runner.yml @@ -41,7 +41,7 @@ jobs: - uses: actions/create-github-app-token@v3 id: app-token with: - app-id: ${{ vars.PLAYWRIGHT_APP_ID }} + client-id: ${{ vars.PLAYWRIGHT_APP_CLIENT_ID }} private-key: ${{ secrets.PLAYWRIGHT_PRIVATE_KEY }} - name: Create Pull Request if: ${{ steps.prepare-branch.outputs.HAS_CHANGES == '1' }} diff --git a/.gitignore b/.gitignore index ee3c339313e6a..df4ab0745589b 100644 --- a/.gitignore +++ b/.gitignore @@ -41,3 +41,4 @@ test-results playwright.env /firefox/ .playwright/ +.npmrc diff --git a/docs/src/api/class-frame.md b/docs/src/api/class-frame.md index 8836c3c5f378e..770da8c509ced 100644 --- a/docs/src/api/class-frame.md +++ b/docs/src/api/class-frame.md @@ -1069,8 +1069,6 @@ Attribute name to get the value for. ### option: Frame.getByRole.description = %%-locator-get-by-role-option-description-%% -### option: Frame.getByRole.busy = %%-locator-get-by-role-option-busy-%% - ## method: Frame.getByTestId * since: v1.27 - returns: <[Locator]> diff --git a/docs/src/api/class-framelocator.md b/docs/src/api/class-framelocator.md index 1305e716ad65e..7f19ceb465ec2 100644 --- a/docs/src/api/class-framelocator.md +++ b/docs/src/api/class-framelocator.md @@ -144,8 +144,6 @@ in that iframe. ### option: FrameLocator.getByRole.description = %%-locator-get-by-role-option-description-%% -### option: FrameLocator.getByRole.busy = %%-locator-get-by-role-option-busy-%% - ## method: FrameLocator.getByTestId * since: v1.27 - returns: <[Locator]> diff --git a/docs/src/api/class-locator.md b/docs/src/api/class-locator.md index bba1f1f9a699a..f9bc16b0d6f94 100644 --- a/docs/src/api/class-locator.md +++ b/docs/src/api/class-locator.md @@ -1488,8 +1488,6 @@ Attribute name to get the value for. ### option: Locator.getByRole.description = %%-locator-get-by-role-option-description-%% -### option: Locator.getByRole.busy = %%-locator-get-by-role-option-busy-%% - ## method: Locator.getByTestId * since: v1.27 - returns: <[Locator]> diff --git a/docs/src/api/class-page.md b/docs/src/api/class-page.md index 913fc56185e9f..4db5da98e309c 100644 --- a/docs/src/api/class-page.md +++ b/docs/src/api/class-page.md @@ -2310,8 +2310,6 @@ Attribute name to get the value for. ### option: Page.getByRole.description = %%-locator-get-by-role-option-description-%% -### option: Page.getByRole.busy = %%-locator-get-by-role-option-busy-%% - ## method: Page.getByTestId * since: v1.27 - returns: <[Locator]> diff --git a/docs/src/api/params.md b/docs/src/api/params.md index 2eec2d6fd3a83..27ec2b1b064b7 100644 --- a/docs/src/api/params.md +++ b/docs/src/api/params.md @@ -1411,14 +1411,6 @@ Whether to find an exact match: case-sensitive and whole-string. Default to fals Required aria role. -## locator-get-by-role-option-busy -* since: v1.61 -- `busy` <[boolean]> - -An attribute that is usually set by `aria-busy`. - -Learn more about [`aria-busy`](https://www.w3.org/TR/wai-aria-1.2/#aria-busy). - ## locator-get-by-role-option-checked * since: v1.27 - `checked` <[boolean]> diff --git a/docs/src/test-api/class-testconfig.md b/docs/src/test-api/class-testconfig.md index 169b8c943f11e..848882fe2ec20 100644 --- a/docs/src/test-api/class-testconfig.md +++ b/docs/src/test-api/class-testconfig.md @@ -517,11 +517,11 @@ export default defineConfig({ ## property: TestConfig.retryStrategy * since: v1.62 -- type: ?<[RetryStrategy]<"immediate"|"deferred">> +- type: ?<[RetryStrategy]<"immediate"|"isolated">> Controls when failed tests are retried. Defaults to `'immediate'`. * `'immediate'` - A failed test is retried as soon as a worker is available, interleaved with the rest of the run. This is the default. -* `'deferred'` - Retries are run only after all tests have had their first attempt, in parallel up to the configured number of [workers](#test-config-workers). +* `'isolated'` - Retries are run at the end, after all other tests have finished, one by one in a single worker. This minimizes the interference between retried tests and the rest of the suite, at the expense of the total run time. Learn more about [test retries](../test-retries.md#retries). @@ -532,7 +532,7 @@ import { defineConfig } from '@playwright/test'; export default defineConfig({ retries: 2, - retryStrategy: 'deferred', + retryStrategy: 'isolated', }); ``` diff --git a/docs/src/test-reporter-api/class-reporter.md b/docs/src/test-reporter-api/class-reporter.md index 67619c914e1f7..68a2a21dab9b5 100644 --- a/docs/src/test-reporter-api/class-reporter.md +++ b/docs/src/test-reporter-api/class-reporter.md @@ -298,25 +298,19 @@ Result of the test run. Whether this reporter uses stdio for reporting. When it does not, Playwright Test could add some output to enhance user experience. If your reporter does not print to the terminal, it is strongly recommended to return `false`. -## optional async method: Reporter.preprocessSuite -* since: v1.61 -- `result` ?<[Object]> - - `implementsSharding` ?<[boolean]> When `true`, Playwright skips its built-in shard filter for this run, leaving sharding to the reporter (typically implemented by calling [`method: TestCase.exclude`] on out-of-shard tests). +## optional async method: Reporter.preprocess +* since: v1.62 Called after the configuration has been resolved and before [`method: Reporter.onBegin`]. Allows a reporter to mark individual tests as skipped, excluded, fixed or failing. -### param: Reporter.preprocessSuite.config -* since: v1.61 -- `config` <[FullConfig]> - -Resolved configuration. - -### param: Reporter.preprocessSuite.suite -* since: v1.61 -- `suite` <[Suite]> +### param: Reporter.preprocess.params +* since: v1.62 +- `params` <[Object]> + - `config` <[FullConfig]> Resolved configuration. + - `suite` <[Suite]> The root suite that contains the projects, files and test cases that will run. + - `testRun` <[TestRun]> Control which tests will run and their expected status. -The root suite that contains the projects, files and test cases that will run. -The suite reflects `--project`, `--grep`/`--grep-invert` and `.only` filtering, so it only contains tests that match the current invocation. Setup and dependency projects are readonly and cannot be excluded from here. +The suite reflects `--project`, `--grep`/`--grep-invert` and `.only` filtering, so it only contains tests that match the current invocation. Setup and dependency projects are readonly and cannot be changed through [TestRun]. -The suite ignores the `--shard` argument: it always contains the full, un-sharded corpus. Playwright applies its built-in sharding after [`method: Reporter.preprocessSuite`] returns, unless the returned `implementsSharding` is `true`. +The suite ignores the `--shard` argument: it always contains the full, un-sharded corpus. Playwright applies its built-in sharding after [`method: Reporter.preprocess`] returns, unless the reporter calls [`method: TestRun.skipSharding`]. diff --git a/docs/src/test-reporter-api/class-suite.md b/docs/src/test-reporter-api/class-suite.md index 9f93dcabd6f4c..1d458c842b3f0 100644 --- a/docs/src/test-reporter-api/class-suite.md +++ b/docs/src/test-reporter-api/class-suite.md @@ -85,41 +85,3 @@ Returns a list of titles from the root down to this suite. Returns the type of the suite. The Suites form the following hierarchy: `root` -> `project` -> `file` -> `describe` -> ...`describe` -> `test`. - -## method: Suite.skip -* since: v1.61 - -Must be called from inside [`method: Reporter.preprocessSuite`]. Mark every [TestCase] of this suite as skipped, see [`method: TestCase.skip`]. - -### param: Suite.skip.reason -* since: v1.61 -- `reason` ?<[string]> - -Optional explanation surfaced as the annotation description. - -## method: Suite.fixme -* since: v1.61 - -Must be called from inside [`method: Reporter.preprocessSuite`]. Mark every [TestCase] of this suite as fixme, see [`method: TestCase.fixme`]. - -### param: Suite.fixme.reason -* since: v1.61 -- `reason` ?<[string]> - -Optional explanation surfaced as the annotation description. - -## method: Suite.fail -* since: v1.61 - -Must be called from inside [`method: Reporter.preprocessSuite`]. Mark every [TestCase] of this suite as expected-to-fail, see [`method: TestCase.fail`]. - -### param: Suite.fail.reason -* since: v1.61 -- `reason` ?<[string]> - -Optional explanation surfaced as the annotation description. - -## method: Suite.exclude -* since: v1.61 - -Must be called from inside [`method: Reporter.preprocessSuite`], exclude this suite from the run. Excluded tests do not appear in the report and their body is not executed. diff --git a/docs/src/test-reporter-api/class-testcase.md b/docs/src/test-reporter-api/class-testcase.md index 95e6052e32ae7..22a8588fb0934 100644 --- a/docs/src/test-reporter-api/class-testcase.md +++ b/docs/src/test-reporter-api/class-testcase.md @@ -107,41 +107,3 @@ Returns a list of titles from the root down to this test. - returns: <[TestCaseType]<"test">> Returns "test". Useful for detecting test cases in [`method: Suite.entries`]. - -## method: TestCase.skip -* since: v1.61 - -Must be called from inside [`method: Reporter.preprocessSuite`], skip this test. The test body is not executed and the test is reported as skipped. - -### param: TestCase.skip.reason -* since: v1.61 -- `reason` ?<[string]> - -Optional explanation surfaced as the annotation description. - -## method: TestCase.fixme -* since: v1.61 - -Must be called from inside [`method: Reporter.preprocessSuite`], mark this test as fixme. The test body is not executed and the test is reported as skipped, with the intention to fix it. - -### param: TestCase.fixme.reason -* since: v1.61 -- `reason` ?<[string]> - -Optional explanation surfaced as the annotation description. - -## method: TestCase.fail -* since: v1.61 - -Must be called from inside [`method: Reporter.preprocessSuite`], mark this test as "should fail". Playwright runs the test and ensures it is actually failing, useful for documenting broken functionality until it is fixed. - -### param: TestCase.fail.reason -* since: v1.61 -- `reason` ?<[string]> - -Optional explanation surfaced as the annotation description. - -## method: TestCase.exclude -* since: v1.61 - -Must be called from inside [`method: Reporter.preprocessSuite`], exclude this test from the run. Excluded tests do not appear in the report and their body is not executed. diff --git a/docs/src/test-reporter-api/class-testrun.md b/docs/src/test-reporter-api/class-testrun.md new file mode 100644 index 0000000000000..563accf59f293 --- /dev/null +++ b/docs/src/test-reporter-api/class-testrun.md @@ -0,0 +1,72 @@ +# class: TestRun +* since: v1.62 +* langs: js + +Controls which tests will run and their expected status. A [TestRun] is available during [`method: Reporter.preprocess`]. Setup and teardown projects cannot be changed. + +## method: TestRun.exclude +* since: v1.62 + +Excludes a test or suite from the run. Excluded tests do not appear in the report and their bodies are not executed. + +### param: TestRun.exclude.test +* since: v1.62 +- `test` <[TestCase]|[Suite]> + +Test or suite to exclude. The root suite cannot be excluded. + +## method: TestRun.fail +* since: v1.62 + +Marks a test or every test in a suite as "should fail". Playwright runs the tests and ensures they are actually failing, useful for documenting broken functionality until it is fixed. + +### param: TestRun.fail.test +* since: v1.62 +- `test` <[TestCase]|[Suite]> + +Test or suite to mark as expected-to-fail. + +### param: TestRun.fail.reason +* since: v1.62 +- `reason` ?<[string]> + +Optional explanation surfaced as the annotation description. + +## method: TestRun.fixme +* since: v1.62 + +Marks a test or every test in a suite as fixme. The test bodies are not executed and the tests are reported as skipped, with the intention to fix them. + +### param: TestRun.fixme.test +* since: v1.62 +- `test` <[TestCase]|[Suite]> + +Test or suite to mark as fixme. + +### param: TestRun.fixme.reason +* since: v1.62 +- `reason` ?<[string]> + +Optional explanation surfaced as the annotation description. + +## method: TestRun.skip +* since: v1.62 + +Skips a test or every test in a suite. The test bodies are not executed and the tests are reported as skipped. + +### param: TestRun.skip.test +* since: v1.62 +- `test` <[TestCase]|[Suite]> + +Test or suite to skip. + +### param: TestRun.skip.reason +* since: v1.62 +- `reason` ?<[string]> + +Optional explanation surfaced as the annotation description. + +## method: TestRun.skipSharding +* since: v1.62 + +Disables Playwright's built-in shard filter for this run, leaving sharding to the reporter. Reporters typically implement their own sharding by calling [`method: TestRun.exclude`] on out-of-shard tests. diff --git a/docs/src/test-reporter-api/class-teststep.md b/docs/src/test-reporter-api/class-teststep.md index 4c312ad2a75c5..0c18e6f768ddf 100644 --- a/docs/src/test-reporter-api/class-teststep.md +++ b/docs/src/test-reporter-api/class-teststep.md @@ -2,7 +2,7 @@ * since: v1.10 * langs: js -Represents a step in the [TestRun]. +Represents a step in a [TestResult]. ## property: TestStep.category * since: v1.10 diff --git a/docs/src/test-reporters-js.md b/docs/src/test-reporters-js.md index 184527e930003..169ba7fae0192 100644 --- a/docs/src/test-reporters-js.md +++ b/docs/src/test-reporters-js.md @@ -275,6 +275,7 @@ HTML report supports the following configuration options and environment variabl | `PLAYWRIGHT_HTML_NO_COPY_PROMPT` | `noCopyPrompt` | If true, disable rendering of the Copy prompt for errors. Supports `true`, `1`, `false`, and `0`. | `false` | `PLAYWRIGHT_HTML_NO_SNIPPETS` | `noSnippets` | If true, disable rendering code snippets in the action log. If there is a top level error, that report section with code snippet will still render. Supports `true`, `1`, `false`, and `0`. | `false` | `PLAYWRIGHT_HTML_DO_NOT_INLINE_ASSETS` | `doNotInlineAssets` | If true, JavaScript, CSS and report data are written as separate files alongside `index.html` instead of being embedded inline. Use this when serving the report under a strict [Content Security Policy](https://developer.mozilla.org/en-US/docs/Web/HTTP/Guides/CSP) that disallows inline scripts and styles. Supports `true`, `1`, `false`, and `0`. | `false` +| `PLAYWRIGHT_HTML_MERGE_FILES` | `mergeFiles` | If true, tests are grouped by their top-level `test.describe()` title instead of the file they belong to. Supports `true`, `1`, `false`, and `0`. | `false` ### Blob reporter diff --git a/packages/html-reporter/src/headerView.tsx b/packages/html-reporter/src/headerView.tsx index ab456e86ca75e..86e8db01492bf 100644 --- a/packages/html-reporter/src/headerView.tsx +++ b/packages/html-reporter/src/headerView.tsx @@ -26,7 +26,6 @@ import { filterWithQuery } from './filter'; import { linkifyText } from '@web/renderUtils'; import { Dialog } from '@web/shared/dialog'; import { kThemeOptions, type Theme, useThemeSetting } from '@web/theme'; -import { useSetting } from '@web/uiUtils'; export const HeaderView: React.FC<{ title: string | undefined, @@ -132,7 +131,6 @@ const SettingsButton: React.FC = () => { const settingsRef = React.useRef(null); const [settingsOpen, setSettingsOpen] = React.useState(false); const [theme, setTheme] = useThemeSetting(); - const [mergeFiles, setMergeFiles] = useSetting('mergeFiles', false); return <>
{ ))} - - ; }; diff --git a/packages/html-reporter/src/reportView.tsx b/packages/html-reporter/src/reportView.tsx index 2eda20a72c78c..c2f3d2fd8ba2d 100644 --- a/packages/html-reporter/src/reportView.tsx +++ b/packages/html-reporter/src/reportView.tsx @@ -26,7 +26,6 @@ import './reportView.css'; import { TestCaseView } from './testCaseView'; import { TestFilesHeader, TestFilesView } from './testFilesView'; import './theme.css'; -import { useSetting } from '@web/uiUtils'; import { Speedboard } from './speedboard'; declare global { @@ -54,7 +53,7 @@ export const ReportView: React.FC<{ const [metadataVisible, setMetadataVisible] = React.useState(false); const [errorsVisible, setErrorsVisible] = React.useState(true); const speedboard = searchParams.has('speedboard'); - const [mergeFiles] = useSetting('mergeFiles', false); + const mergeFiles = !!report?.json()?.options.mergeFiles; const testId = searchParams.get('testId'); const q = searchParams.get('q')?.toString() || ''; const filterParam = q ? '&q=' + q : ''; diff --git a/packages/html-reporter/src/types.d.ts b/packages/html-reporter/src/types.d.ts index ca2ecec64dd3b..62bab6c0e3a4c 100644 --- a/packages/html-reporter/src/types.d.ts +++ b/packages/html-reporter/src/types.d.ts @@ -40,6 +40,7 @@ export type HTMLReportOptions = { title?: string; noCopyPrompt?: boolean; noSnippets?: boolean; + mergeFiles?: boolean; }; export type HTMLReport = { diff --git a/packages/injected/src/roleSelectorEngine.ts b/packages/injected/src/roleSelectorEngine.ts index 7d18a013d74b7..ad3c3a5ac06fc 100644 --- a/packages/injected/src/roleSelectorEngine.ts +++ b/packages/injected/src/roleSelectorEngine.ts @@ -17,7 +17,7 @@ import { parseAttributeSelector } from '@isomorphic/selectorParser'; import { normalizeWhiteSpace } from '@isomorphic/stringUtils'; -import { beginAriaCaches, endAriaCaches, getAriaBusy, getAriaChecked, getAriaDisabled, getAriaExpanded, getAriaLevel, getAriaPressed, getAriaRole, getAriaSelected, getElementAccessibleDescription, getElementAccessibleNameText, isElementHiddenForAria, kAriaCheckedRoles, kAriaExpandedRoles, kAriaLevelRoles, kAriaPressedRoles, kAriaSelectedRoles } from './roleUtils'; +import { beginAriaCaches, endAriaCaches, getAriaChecked, getAriaDisabled, getAriaExpanded, getAriaLevel, getAriaPressed, getAriaRole, getAriaSelected, getElementAccessibleDescription, getElementAccessibleNameText, isElementHiddenForAria, kAriaCheckedRoles, kAriaExpandedRoles, kAriaLevelRoles, kAriaPressedRoles, kAriaSelectedRoles } from './roleUtils'; import { matchesAttributePart } from './selectorUtils'; import type { AttributeSelectorOperator, AttributeSelectorPart } from '@isomorphic/selectorParser'; @@ -37,11 +37,10 @@ type RoleEngineOptions = { expanded?: boolean; level?: number; disabled?: boolean; - busy?: boolean; includeHidden?: boolean; }; -const kSupportedAttributes = ['selected', 'checked', 'pressed', 'expanded', 'level', 'disabled', 'busy', 'name', 'description', 'include-hidden']; +const kSupportedAttributes = ['selected', 'checked', 'pressed', 'expanded', 'level', 'disabled', 'name', 'description', 'include-hidden']; kSupportedAttributes.sort(); function validateSupportedRole(attr: string, roles: string[], role: string) { @@ -107,12 +106,6 @@ function validateAttributes(attrs: AttributeSelectorPart[], role: string): RoleE options.disabled = attr.op === '' ? true : attr.value; break; } - case 'busy': { - validateSupportedValues(attr, [true, false]); - validateSupportedOp(attr, ['', '=']); - options.busy = attr.op === '' ? true : attr.value; - break; - } case 'name': { if (attr.op === '') throw new Error(`"name" attribute must have a value`); @@ -164,8 +157,6 @@ function queryRole(scope: SelectorRoot, options: RoleEngineOptions, internal: bo return; if (options.disabled !== undefined && getAriaDisabled(element) !== options.disabled) return; - if (options.busy !== undefined && getAriaBusy(element) !== options.busy) - return; if (!options.includeHidden) { const isHidden = isElementHiddenForAria(element); if (isHidden) diff --git a/packages/injected/src/roleUtils.ts b/packages/injected/src/roleUtils.ts index b167f354a666e..e518b76c43208 100644 --- a/packages/injected/src/roleUtils.ts +++ b/packages/injected/src/roleUtils.ts @@ -1179,12 +1179,6 @@ function hasExplicitAriaDisabled(element: Element | undefined, isAncestor = fals return false; } -export function getAriaBusy(element: Element): boolean { - // https://www.w3.org/TR/wai-aria-1.2/#aria-busy - // aria-busy is a global state with a default value of "false". - return getAriaBoolean(element.getAttribute('aria-busy')) === true; -} - function getAccessibleNameFromAssociatedLabels(labels: Iterable, options: AccessibleNameOptions): CompositeString { return joinCompositeString([...labels].map(label => getTextAlternativeInternal(label, { ...options, diff --git a/packages/isomorphic/locatorUtils.ts b/packages/isomorphic/locatorUtils.ts index 0708cbccde3f5..77cb2d715a7ef 100644 --- a/packages/isomorphic/locatorUtils.ts +++ b/packages/isomorphic/locatorUtils.ts @@ -17,7 +17,6 @@ import { escapeForAttributeSelector, escapeForTextSelector } from './stringUtils'; export type ByRoleOptions = { - busy?: boolean; checked?: boolean; description?: string | RegExp; disabled?: boolean; @@ -87,7 +86,5 @@ export function getByRoleSelector(role: string, options: ByRoleOptions = {}): st props.push(['description', escapeForAttributeSelector(options.description, !!options.exact)]); if (options.pressed !== undefined) props.push(['pressed', String(options.pressed)]); - if (options.busy !== undefined) - props.push(['busy', String(options.busy)]); return `internal:role=${role}${props.map(([n, v]) => `[${n}=${v}]`).join('')}`; } diff --git a/packages/playwright-client/types/types.d.ts b/packages/playwright-client/types/types.d.ts index ed7f5c099bee9..e0f0b6486db84 100644 --- a/packages/playwright-client/types/types.d.ts +++ b/packages/playwright-client/types/types.d.ts @@ -3080,13 +3080,6 @@ export interface Page { * @param options */ getByRole(role: "alert"|"alertdialog"|"application"|"article"|"banner"|"blockquote"|"button"|"caption"|"cell"|"checkbox"|"code"|"columnheader"|"combobox"|"complementary"|"contentinfo"|"definition"|"deletion"|"dialog"|"directory"|"document"|"emphasis"|"feed"|"figure"|"form"|"generic"|"grid"|"gridcell"|"group"|"heading"|"img"|"insertion"|"link"|"list"|"listbox"|"listitem"|"log"|"main"|"marquee"|"math"|"meter"|"menu"|"menubar"|"menuitem"|"menuitemcheckbox"|"menuitemradio"|"navigation"|"none"|"note"|"option"|"paragraph"|"presentation"|"progressbar"|"radio"|"radiogroup"|"region"|"row"|"rowgroup"|"rowheader"|"scrollbar"|"search"|"searchbox"|"separator"|"slider"|"spinbutton"|"status"|"strong"|"subscript"|"superscript"|"switch"|"tab"|"table"|"tablist"|"tabpanel"|"term"|"textbox"|"time"|"timer"|"toolbar"|"tooltip"|"tree"|"treegrid"|"treeitem", options?: { - /** - * An attribute that is usually set by `aria-busy`. - * - * Learn more about [`aria-busy`](https://www.w3.org/TR/wai-aria-1.2/#aria-busy). - */ - busy?: boolean; - /** * An attribute that is usually set by `aria-checked` or native `` controls. * @@ -7379,13 +7372,6 @@ export interface Frame { * @param options */ getByRole(role: "alert"|"alertdialog"|"application"|"article"|"banner"|"blockquote"|"button"|"caption"|"cell"|"checkbox"|"code"|"columnheader"|"combobox"|"complementary"|"contentinfo"|"definition"|"deletion"|"dialog"|"directory"|"document"|"emphasis"|"feed"|"figure"|"form"|"generic"|"grid"|"gridcell"|"group"|"heading"|"img"|"insertion"|"link"|"list"|"listbox"|"listitem"|"log"|"main"|"marquee"|"math"|"meter"|"menu"|"menubar"|"menuitem"|"menuitemcheckbox"|"menuitemradio"|"navigation"|"none"|"note"|"option"|"paragraph"|"presentation"|"progressbar"|"radio"|"radiogroup"|"region"|"row"|"rowgroup"|"rowheader"|"scrollbar"|"search"|"searchbox"|"separator"|"slider"|"spinbutton"|"status"|"strong"|"subscript"|"superscript"|"switch"|"tab"|"table"|"tablist"|"tabpanel"|"term"|"textbox"|"time"|"timer"|"toolbar"|"tooltip"|"tree"|"treegrid"|"treeitem", options?: { - /** - * An attribute that is usually set by `aria-busy`. - * - * Learn more about [`aria-busy`](https://www.w3.org/TR/wai-aria-1.2/#aria-busy). - */ - busy?: boolean; - /** * An attribute that is usually set by `aria-checked` or native `` controls. * @@ -15262,13 +15248,6 @@ export interface Locator { * @param options */ getByRole(role: "alert"|"alertdialog"|"application"|"article"|"banner"|"blockquote"|"button"|"caption"|"cell"|"checkbox"|"code"|"columnheader"|"combobox"|"complementary"|"contentinfo"|"definition"|"deletion"|"dialog"|"directory"|"document"|"emphasis"|"feed"|"figure"|"form"|"generic"|"grid"|"gridcell"|"group"|"heading"|"img"|"insertion"|"link"|"list"|"listbox"|"listitem"|"log"|"main"|"marquee"|"math"|"meter"|"menu"|"menubar"|"menuitem"|"menuitemcheckbox"|"menuitemradio"|"navigation"|"none"|"note"|"option"|"paragraph"|"presentation"|"progressbar"|"radio"|"radiogroup"|"region"|"row"|"rowgroup"|"rowheader"|"scrollbar"|"search"|"searchbox"|"separator"|"slider"|"spinbutton"|"status"|"strong"|"subscript"|"superscript"|"switch"|"tab"|"table"|"tablist"|"tabpanel"|"term"|"textbox"|"time"|"timer"|"toolbar"|"tooltip"|"tree"|"treegrid"|"treeitem", options?: { - /** - * An attribute that is usually set by `aria-busy`. - * - * Learn more about [`aria-busy`](https://www.w3.org/TR/wai-aria-1.2/#aria-busy). - */ - busy?: boolean; - /** * An attribute that is usually set by `aria-checked` or native `` controls. * @@ -21077,13 +21056,6 @@ export interface FrameLocator { * @param options */ getByRole(role: "alert"|"alertdialog"|"application"|"article"|"banner"|"blockquote"|"button"|"caption"|"cell"|"checkbox"|"code"|"columnheader"|"combobox"|"complementary"|"contentinfo"|"definition"|"deletion"|"dialog"|"directory"|"document"|"emphasis"|"feed"|"figure"|"form"|"generic"|"grid"|"gridcell"|"group"|"heading"|"img"|"insertion"|"link"|"list"|"listbox"|"listitem"|"log"|"main"|"marquee"|"math"|"meter"|"menu"|"menubar"|"menuitem"|"menuitemcheckbox"|"menuitemradio"|"navigation"|"none"|"note"|"option"|"paragraph"|"presentation"|"progressbar"|"radio"|"radiogroup"|"region"|"row"|"rowgroup"|"rowheader"|"scrollbar"|"search"|"searchbox"|"separator"|"slider"|"spinbutton"|"status"|"strong"|"subscript"|"superscript"|"switch"|"tab"|"table"|"tablist"|"tabpanel"|"term"|"textbox"|"time"|"timer"|"toolbar"|"tooltip"|"tree"|"treegrid"|"treeitem", options?: { - /** - * An attribute that is usually set by `aria-busy`. - * - * Learn more about [`aria-busy`](https://www.w3.org/TR/wai-aria-1.2/#aria-busy). - */ - busy?: boolean; - /** * An attribute that is usually set by `aria-checked` or native `` controls. * diff --git a/packages/playwright-core/browsers.json b/packages/playwright-core/browsers.json index cdb17ffb3e102..1e6400f84e2a7 100644 --- a/packages/playwright-core/browsers.json +++ b/packages/playwright-core/browsers.json @@ -50,8 +50,6 @@ "revisionOverrides": { "mac14": "2251", "mac14-arm64": "2251", - "debian11-x64": "2105", - "debian11-arm64": "2105", "ubuntu20.04-x64": "2092", "ubuntu20.04-arm64": "2092" }, @@ -61,11 +59,7 @@ { "name": "ffmpeg", "revision": "1011", - "installByDefault": true, - "revisionOverrides": { - "mac12": "1010", - "mac12-arm64": "1010" - } + "installByDefault": true }, { "name": "winldd", diff --git a/packages/playwright-core/src/server/registry/index.ts b/packages/playwright-core/src/server/registry/index.ts index 1d843a9ff7ae8..2b6e0141a7689 100644 --- a/packages/playwright-core/src/server/registry/index.ts +++ b/packages/playwright-core/src/server/registry/index.ts @@ -155,21 +155,21 @@ const DOWNLOAD_PATHS: Record = { 'ubuntu22.04-arm64': 'builds/chromium/%s/chromium-linux-arm64.zip', 'ubuntu24.04-arm64': 'builds/chromium/%s/chromium-linux-arm64.zip', 'ubuntu26.04-arm64': 'builds/chromium/%s/chromium-linux-arm64.zip', - 'debian11-x64': cftUrl('linux64/chrome-linux64.zip'), - 'debian11-arm64': 'builds/chromium/%s/chromium-linux-arm64.zip', + 'debian11-x64': undefined, + 'debian11-arm64': undefined, 'debian12-x64': cftUrl('linux64/chrome-linux64.zip'), 'debian12-arm64': 'builds/chromium/%s/chromium-linux-arm64.zip', 'debian13-x64': cftUrl('linux64/chrome-linux64.zip'), 'debian13-arm64': 'builds/chromium/%s/chromium-linux-arm64.zip', - 'mac10.13': cftUrl('mac-x64/chrome-mac-x64.zip'), - 'mac10.14': cftUrl('mac-x64/chrome-mac-x64.zip'), - 'mac10.15': cftUrl('mac-x64/chrome-mac-x64.zip'), - 'mac11': cftUrl('mac-x64/chrome-mac-x64.zip'), - 'mac11-arm64': cftUrl('mac-arm64/chrome-mac-arm64.zip'), - 'mac12': cftUrl('mac-x64/chrome-mac-x64.zip'), - 'mac12-arm64': cftUrl('mac-arm64/chrome-mac-arm64.zip'), - 'mac13': cftUrl('mac-x64/chrome-mac-x64.zip'), - 'mac13-arm64': cftUrl('mac-arm64/chrome-mac-arm64.zip'), + 'mac10.13': undefined, + 'mac10.14': undefined, + 'mac10.15': undefined, + 'mac11': undefined, + 'mac11-arm64': undefined, + 'mac12': undefined, + 'mac12-arm64': undefined, + 'mac13': undefined, + 'mac13-arm64': undefined, 'mac14': cftUrl('mac-x64/chrome-mac-x64.zip'), 'mac14-arm64': cftUrl('mac-arm64/chrome-mac-arm64.zip'), 'mac15': cftUrl('mac-x64/chrome-mac-x64.zip'), @@ -190,8 +190,8 @@ const DOWNLOAD_PATHS: Record = { 'ubuntu22.04-arm64': 'builds/chromium/%s/chromium-headless-shell-linux-arm64.zip', 'ubuntu24.04-arm64': 'builds/chromium/%s/chromium-headless-shell-linux-arm64.zip', 'ubuntu26.04-arm64': 'builds/chromium/%s/chromium-headless-shell-linux-arm64.zip', - 'debian11-x64': cftUrl('linux64/chrome-headless-shell-linux64.zip'), - 'debian11-arm64': 'builds/chromium/%s/chromium-headless-shell-linux-arm64.zip', + 'debian11-x64': undefined, + 'debian11-arm64': undefined, 'debian12-x64': cftUrl('linux64/chrome-headless-shell-linux64.zip'), 'debian12-arm64': 'builds/chromium/%s/chromium-headless-shell-linux-arm64.zip', 'debian13-x64': cftUrl('linux64/chrome-headless-shell-linux64.zip'), @@ -199,12 +199,12 @@ const DOWNLOAD_PATHS: Record = { 'mac10.13': undefined, 'mac10.14': undefined, 'mac10.15': undefined, - 'mac11': cftUrl('mac-x64/chrome-headless-shell-mac-x64.zip'), - 'mac11-arm64': cftUrl('mac-arm64/chrome-headless-shell-mac-arm64.zip'), - 'mac12': cftUrl('mac-x64/chrome-headless-shell-mac-x64.zip'), - 'mac12-arm64': cftUrl('mac-arm64/chrome-headless-shell-mac-arm64.zip'), - 'mac13': cftUrl('mac-x64/chrome-headless-shell-mac-x64.zip'), - 'mac13-arm64': cftUrl('mac-arm64/chrome-headless-shell-mac-arm64.zip'), + 'mac11': undefined, + 'mac11-arm64': undefined, + 'mac12': undefined, + 'mac12-arm64': undefined, + 'mac13': undefined, + 'mac13-arm64': undefined, 'mac14': cftUrl('mac-x64/chrome-headless-shell-mac-x64.zip'), 'mac14-arm64': cftUrl('mac-arm64/chrome-headless-shell-mac-arm64.zip'), 'mac15': cftUrl('mac-x64/chrome-headless-shell-mac-x64.zip'), @@ -225,21 +225,21 @@ const DOWNLOAD_PATHS: Record = { 'ubuntu22.04-arm64': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-linux-arm64.zip', 'ubuntu24.04-arm64': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-linux-arm64.zip', 'ubuntu26.04-arm64': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-linux-arm64.zip', - 'debian11-x64': cftUrl('linux64/chrome-linux64.zip'), - 'debian11-arm64': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-linux-arm64.zip', + 'debian11-x64': undefined, + 'debian11-arm64': undefined, 'debian12-x64': cftUrl('linux64/chrome-linux64.zip'), 'debian12-arm64': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-linux-arm64.zip', 'debian13-x64': cftUrl('linux64/chrome-linux64.zip'), 'debian13-arm64': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-linux-arm64.zip', - 'mac10.13': cftUrl('mac-x64/chrome-mac-x64.zip'), - 'mac10.14': cftUrl('mac-x64/chrome-mac-x64.zip'), - 'mac10.15': cftUrl('mac-x64/chrome-mac-x64.zip'), - 'mac11': cftUrl('mac-x64/chrome-mac-x64.zip'), - 'mac11-arm64': cftUrl('mac-arm64/chrome-mac-arm64.zip'), - 'mac12': cftUrl('mac-x64/chrome-mac-x64.zip'), - 'mac12-arm64': cftUrl('mac-arm64/chrome-mac-arm64.zip'), - 'mac13': cftUrl('mac-x64/chrome-mac-x64.zip'), - 'mac13-arm64': cftUrl('mac-arm64/chrome-mac-arm64.zip'), + 'mac10.13': undefined, + 'mac10.14': undefined, + 'mac10.15': undefined, + 'mac11': undefined, + 'mac11-arm64': undefined, + 'mac12': undefined, + 'mac12-arm64': undefined, + 'mac13': undefined, + 'mac13-arm64': undefined, 'mac14': cftUrl('mac-x64/chrome-mac-x64.zip'), 'mac14-arm64': cftUrl('mac-arm64/chrome-mac-arm64.zip'), 'mac15': cftUrl('mac-x64/chrome-mac-x64.zip'), @@ -260,8 +260,8 @@ const DOWNLOAD_PATHS: Record = { 'ubuntu22.04-arm64': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-headless-shell-linux-arm64.zip', 'ubuntu24.04-arm64': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-headless-shell-linux-arm64.zip', 'ubuntu26.04-arm64': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-headless-shell-linux-arm64.zip', - 'debian11-x64': cftUrl('linux64/chrome-headless-shell-linux64.zip'), - 'debian11-arm64': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-headless-shell-linux-arm64.zip', + 'debian11-x64': undefined, + 'debian11-arm64': undefined, 'debian12-x64': cftUrl('linux64/chrome-headless-shell-linux64.zip'), 'debian12-arm64': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-headless-shell-linux-arm64.zip', 'debian13-x64': cftUrl('linux64/chrome-headless-shell-linux64.zip'), @@ -269,12 +269,12 @@ const DOWNLOAD_PATHS: Record = { 'mac10.13': undefined, 'mac10.14': undefined, 'mac10.15': undefined, - 'mac11': cftUrl('mac-x64/chrome-headless-shell-mac-x64.zip'), - 'mac11-arm64': cftUrl('mac-arm64/chrome-headless-shell-mac-arm64.zip'), - 'mac12': cftUrl('mac-x64/chrome-headless-shell-mac-x64.zip'), - 'mac12-arm64': cftUrl('mac-arm64/chrome-headless-shell-mac-arm64.zip'), - 'mac13': cftUrl('mac-x64/chrome-headless-shell-mac-x64.zip'), - 'mac13-arm64': cftUrl('mac-arm64/chrome-headless-shell-mac-arm64.zip'), + 'mac11': undefined, + 'mac11-arm64': undefined, + 'mac12': undefined, + 'mac12-arm64': undefined, + 'mac13': undefined, + 'mac13-arm64': undefined, 'mac14': cftUrl('mac-x64/chrome-headless-shell-mac-x64.zip'), 'mac14-arm64': cftUrl('mac-arm64/chrome-headless-shell-mac-arm64.zip'), 'mac15': cftUrl('mac-x64/chrome-headless-shell-mac-x64.zip'), @@ -295,21 +295,21 @@ const DOWNLOAD_PATHS: Record = { 'ubuntu22.04-arm64': 'builds/firefox/%s/firefox-ubuntu-22.04-arm64.zip', 'ubuntu24.04-arm64': 'builds/firefox/%s/firefox-ubuntu-24.04-arm64.zip', 'ubuntu26.04-arm64': 'builds/firefox/%s/firefox-ubuntu-24.04-arm64.zip', - 'debian11-x64': 'builds/firefox/%s/firefox-debian-11.zip', - 'debian11-arm64': 'builds/firefox/%s/firefox-debian-11-arm64.zip', + 'debian11-x64': undefined, + 'debian11-arm64': undefined, 'debian12-x64': 'builds/firefox/%s/firefox-debian-12.zip', 'debian12-arm64': 'builds/firefox/%s/firefox-debian-12-arm64.zip', 'debian13-x64': 'builds/firefox/%s/firefox-debian-13.zip', 'debian13-arm64': 'builds/firefox/%s/firefox-debian-13-arm64.zip', - 'mac10.13': 'builds/firefox/%s/firefox-mac.zip', - 'mac10.14': 'builds/firefox/%s/firefox-mac.zip', - 'mac10.15': 'builds/firefox/%s/firefox-mac.zip', - 'mac11': 'builds/firefox/%s/firefox-mac.zip', - 'mac11-arm64': 'builds/firefox/%s/firefox-mac-arm64.zip', - 'mac12': 'builds/firefox/%s/firefox-mac.zip', - 'mac12-arm64': 'builds/firefox/%s/firefox-mac-arm64.zip', - 'mac13': 'builds/firefox/%s/firefox-mac.zip', - 'mac13-arm64': 'builds/firefox/%s/firefox-mac-arm64.zip', + 'mac10.13': undefined, + 'mac10.14': undefined, + 'mac10.15': undefined, + 'mac11': undefined, + 'mac11-arm64': undefined, + 'mac12': undefined, + 'mac12-arm64': undefined, + 'mac13': undefined, + 'mac13-arm64': undefined, 'mac14': 'builds/firefox/%s/firefox-mac.zip', 'mac14-arm64': 'builds/firefox/%s/firefox-mac-arm64.zip', 'mac15': 'builds/firefox/%s/firefox-mac.zip', @@ -330,21 +330,21 @@ const DOWNLOAD_PATHS: Record = { 'ubuntu22.04-arm64': 'builds/firefox-beta/%s/firefox-beta-ubuntu-22.04-arm64.zip', 'ubuntu24.04-arm64': 'builds/firefox-beta/%s/firefox-beta-ubuntu-24.04-arm64.zip', 'ubuntu26.04-arm64': 'builds/firefox-beta/%s/firefox-beta-ubuntu-24.04-arm64.zip', - 'debian11-x64': 'builds/firefox-beta/%s/firefox-beta-debian-11.zip', - 'debian11-arm64': 'builds/firefox-beta/%s/firefox-beta-debian-11-arm64.zip', + 'debian11-x64': undefined, + 'debian11-arm64': undefined, 'debian12-x64': 'builds/firefox-beta/%s/firefox-beta-debian-12.zip', 'debian12-arm64': 'builds/firefox-beta/%s/firefox-beta-debian-12-arm64.zip', 'debian13-x64': 'builds/firefox-beta/%s/firefox-beta-debian-12.zip', 'debian13-arm64': 'builds/firefox-beta/%s/firefox-beta-debian-12-arm64.zip', - 'mac10.13': 'builds/firefox-beta/%s/firefox-beta-mac.zip', - 'mac10.14': 'builds/firefox-beta/%s/firefox-beta-mac.zip', - 'mac10.15': 'builds/firefox-beta/%s/firefox-beta-mac.zip', - 'mac11': 'builds/firefox-beta/%s/firefox-beta-mac.zip', - 'mac11-arm64': 'builds/firefox-beta/%s/firefox-beta-mac-arm64.zip', - 'mac12': 'builds/firefox-beta/%s/firefox-beta-mac.zip', - 'mac12-arm64': 'builds/firefox-beta/%s/firefox-beta-mac-arm64.zip', - 'mac13': 'builds/firefox-beta/%s/firefox-beta-mac.zip', - 'mac13-arm64': 'builds/firefox-beta/%s/firefox-beta-mac-arm64.zip', + 'mac10.13': undefined, + 'mac10.14': undefined, + 'mac10.15': undefined, + 'mac11': undefined, + 'mac11-arm64': undefined, + 'mac12': undefined, + 'mac12-arm64': undefined, + 'mac13': undefined, + 'mac13-arm64': undefined, 'mac14': 'builds/firefox-beta/%s/firefox-beta-mac.zip', 'mac14-arm64': 'builds/firefox-beta/%s/firefox-beta-mac-arm64.zip', 'mac15': 'builds/firefox-beta/%s/firefox-beta-mac.zip', @@ -365,8 +365,8 @@ const DOWNLOAD_PATHS: Record = { 'ubuntu22.04-arm64': 'builds/webkit/%s/webkit-ubuntu-22.04-arm64.zip', 'ubuntu24.04-arm64': 'builds/webkit/%s/webkit-ubuntu-24.04-arm64.zip', 'ubuntu26.04-arm64': 'builds/webkit/%s/webkit-ubuntu-26.04-arm64.zip', - 'debian11-x64': 'builds/webkit/%s/webkit-debian-11.zip', - 'debian11-arm64': 'builds/webkit/%s/webkit-debian-11-arm64.zip', + 'debian11-x64': undefined, + 'debian11-arm64': undefined, 'debian12-x64': 'builds/webkit/%s/webkit-debian-12.zip', 'debian12-arm64': 'builds/webkit/%s/webkit-debian-12-arm64.zip', 'debian13-x64': 'builds/webkit/%s/webkit-debian-13.zip', @@ -400,21 +400,21 @@ const DOWNLOAD_PATHS: Record = { 'ubuntu22.04-arm64': 'builds/ffmpeg/%s/ffmpeg-linux-arm64.zip', 'ubuntu24.04-arm64': 'builds/ffmpeg/%s/ffmpeg-linux-arm64.zip', 'ubuntu26.04-arm64': 'builds/ffmpeg/%s/ffmpeg-linux-arm64.zip', - 'debian11-x64': 'builds/ffmpeg/%s/ffmpeg-linux.zip', - 'debian11-arm64': 'builds/ffmpeg/%s/ffmpeg-linux-arm64.zip', + 'debian11-x64': undefined, + 'debian11-arm64': undefined, 'debian12-x64': 'builds/ffmpeg/%s/ffmpeg-linux.zip', 'debian12-arm64': 'builds/ffmpeg/%s/ffmpeg-linux-arm64.zip', 'debian13-x64': 'builds/ffmpeg/%s/ffmpeg-linux.zip', 'debian13-arm64': 'builds/ffmpeg/%s/ffmpeg-linux-arm64.zip', - 'mac10.13': 'builds/ffmpeg/%s/ffmpeg-mac.zip', - 'mac10.14': 'builds/ffmpeg/%s/ffmpeg-mac.zip', - 'mac10.15': 'builds/ffmpeg/%s/ffmpeg-mac.zip', - 'mac11': 'builds/ffmpeg/%s/ffmpeg-mac.zip', - 'mac11-arm64': 'builds/ffmpeg/%s/ffmpeg-mac-arm64.zip', - 'mac12': 'builds/ffmpeg/%s/ffmpeg-mac.zip', - 'mac12-arm64': 'builds/ffmpeg/%s/ffmpeg-mac-arm64.zip', - 'mac13': 'builds/ffmpeg/%s/ffmpeg-mac.zip', - 'mac13-arm64': 'builds/ffmpeg/%s/ffmpeg-mac-arm64.zip', + 'mac10.13': undefined, + 'mac10.14': undefined, + 'mac10.15': undefined, + 'mac11': undefined, + 'mac11-arm64': undefined, + 'mac12': undefined, + 'mac12-arm64': undefined, + 'mac13': undefined, + 'mac13-arm64': undefined, 'mac14': 'builds/ffmpeg/%s/ffmpeg-mac.zip', 'mac14-arm64': 'builds/ffmpeg/%s/ffmpeg-mac-arm64.zip', 'mac15': 'builds/ffmpeg/%s/ffmpeg-mac.zip', @@ -470,21 +470,21 @@ const DOWNLOAD_PATHS: Record = { 'ubuntu22.04-arm64': 'builds/android/%s/android.zip', 'ubuntu24.04-arm64': 'builds/android/%s/android.zip', 'ubuntu26.04-arm64': 'builds/android/%s/android.zip', - 'debian11-x64': 'builds/android/%s/android.zip', - 'debian11-arm64': 'builds/android/%s/android.zip', + 'debian11-x64': undefined, + 'debian11-arm64': undefined, 'debian12-x64': 'builds/android/%s/android.zip', 'debian12-arm64': 'builds/android/%s/android.zip', 'debian13-x64': 'builds/android/%s/android.zip', 'debian13-arm64': 'builds/android/%s/android.zip', - 'mac10.13': 'builds/android/%s/android.zip', - 'mac10.14': 'builds/android/%s/android.zip', - 'mac10.15': 'builds/android/%s/android.zip', - 'mac11': 'builds/android/%s/android.zip', - 'mac11-arm64': 'builds/android/%s/android.zip', - 'mac12': 'builds/android/%s/android.zip', - 'mac12-arm64': 'builds/android/%s/android.zip', - 'mac13': 'builds/android/%s/android.zip', - 'mac13-arm64': 'builds/android/%s/android.zip', + 'mac10.13': undefined, + 'mac10.14': undefined, + 'mac10.15': undefined, + 'mac11': undefined, + 'mac11-arm64': undefined, + 'mac12': undefined, + 'mac12-arm64': undefined, + 'mac13': undefined, + 'mac13-arm64': undefined, 'mac14': 'builds/android/%s/android.zip', 'mac14-arm64': 'builds/android/%s/android.zip', 'mac15': 'builds/android/%s/android.zip', diff --git a/packages/playwright-core/src/server/registry/nativeDeps.ts b/packages/playwright-core/src/server/registry/nativeDeps.ts index 7cb4583c2a912..5e02aabbafd57 100644 --- a/packages/playwright-core/src/server/registry/nativeDeps.ts +++ b/packages/playwright-core/src/server/registry/nativeDeps.ts @@ -883,217 +883,6 @@ export const deps: any = { }, }, - 'debian11-x64': { - tools: [ - 'xvfb', - 'fonts-noto-color-emoji', - 'fonts-unifont', - 'libfontconfig1', - 'libfreetype6', - 'xfonts-cyrillic', - 'xfonts-scalable', - 'fonts-liberation', - 'fonts-ipafont-gothic', - 'fonts-wqy-zenhei', - 'fonts-tlwg-loma-otf', - 'fonts-freefont-ttf', - ], - chromium: [ - 'libasound2', - 'libatk-bridge2.0-0', - 'libatk1.0-0', - 'libatspi2.0-0', - 'libcairo2', - 'libcups2', - 'libdbus-1-3', - 'libdrm2', - 'libgbm1', - 'libglib2.0-0', - 'libnspr4', - 'libnss3', - 'libpango-1.0-0', - 'libwayland-client0', - 'libx11-6', - 'libxcb1', - 'libxcomposite1', - 'libxdamage1', - 'libxext6', - 'libxfixes3', - 'libxkbcommon0', - 'libxrandr2' - ], - firefox: [ - 'libasound2', - 'libatk1.0-0', - 'libavcodec58', - 'libcairo-gobject2', - 'libcairo2', - 'libdbus-1-3', - 'libdbus-glib-1-2', - 'libfontconfig1', - 'libfreetype6', - 'libgdk-pixbuf-2.0-0', - 'libglib2.0-0', - 'libgtk-3-0', - 'libharfbuzz0b', - 'libpango-1.0-0', - 'libpangocairo-1.0-0', - 'libx11-6', - 'libx11-xcb1', - 'libxcb-shm0', - 'libxcb1', - 'libxcomposite1', - 'libxcursor1', - 'libxdamage1', - 'libxext6', - 'libxfixes3', - 'libxi6', - 'libxrandr2', - 'libxrender1', - 'libxtst6' - ], - webkit: [ - 'gstreamer1.0-libav', - 'gstreamer1.0-plugins-bad', - 'gstreamer1.0-plugins-base', - 'gstreamer1.0-plugins-good', - 'libatk-bridge2.0-0', - 'libatk1.0-0', - 'libcairo2', - 'libdbus-1-3', - 'libdrm2', - 'libegl1', - 'libenchant-2-2', - 'libepoxy0', - 'libevdev2', - 'libfontconfig1', - 'libfreetype6', - 'libgbm1', - 'libgdk-pixbuf-2.0-0', - 'libgles2', - 'libglib2.0-0', - 'libglx0', - 'libgstreamer-gl1.0-0', - 'libgstreamer-plugins-base1.0-0', - 'libgstreamer1.0-0', - 'libgtk-3-0', - 'libgudev-1.0-0', - 'libharfbuzz-icu0', - 'libharfbuzz0b', - 'libhyphen0', - 'libicu67', - 'libjpeg62-turbo', - 'liblcms2-2', - 'libmanette-0.2-0', - 'libnghttp2-14', - 'libnotify4', - 'libopengl0', - 'libopenjp2-7', - 'libopus0', - 'libpango-1.0-0', - 'libpng16-16', - 'libproxy1v5', - 'libsecret-1-0', - 'libwayland-client0', - 'libwayland-egl1', - 'libwayland-server0', - 'libwebp6', - 'libwebpdemux2', - 'libwoff1', - 'libx11-6', - 'libxcomposite1', - 'libxdamage1', - 'libxkbcommon0', - 'libxml2', - 'libxslt1.1', - 'libatomic1', - 'libevent-2.1-7', - ], - lib2package: { - 'libasound.so.2': 'libasound2', - 'libatk-1.0.so.0': 'libatk1.0-0', - 'libatk-bridge-2.0.so.0': 'libatk-bridge2.0-0', - 'libatspi.so.0': 'libatspi2.0-0', - 'libcairo-gobject.so.2': 'libcairo-gobject2', - 'libcairo.so.2': 'libcairo2', - 'libcups.so.2': 'libcups2', - 'libdbus-1.so.3': 'libdbus-1-3', - 'libdbus-glib-1.so.2': 'libdbus-glib-1-2', - 'libdrm.so.2': 'libdrm2', - 'libEGL.so.1': 'libegl1', - 'libenchant-2.so.2': 'libenchant-2-2', - 'libepoxy.so.0': 'libepoxy0', - 'libevdev.so.2': 'libevdev2', - 'libfontconfig.so.1': 'libfontconfig1', - 'libfreetype.so.6': 'libfreetype6', - 'libgbm.so.1': 'libgbm1', - 'libgdk_pixbuf-2.0.so.0': 'libgdk-pixbuf-2.0-0', - 'libgdk-3.so.0': 'libgtk-3-0', - 'libgio-2.0.so.0': 'libglib2.0-0', - 'libGLESv2.so.2': 'libgles2', - 'libglib-2.0.so.0': 'libglib2.0-0', - 'libGLX.so.0': 'libglx0', - 'libgmodule-2.0.so.0': 'libglib2.0-0', - 'libgobject-2.0.so.0': 'libglib2.0-0', - 'libgstallocators-1.0.so.0': 'libgstreamer-plugins-base1.0-0', - 'libgstapp-1.0.so.0': 'libgstreamer-plugins-base1.0-0', - 'libgstaudio-1.0.so.0': 'libgstreamer-plugins-base1.0-0', - 'libgstbase-1.0.so.0': 'libgstreamer1.0-0', - 'libgstfft-1.0.so.0': 'libgstreamer-plugins-base1.0-0', - 'libgstgl-1.0.so.0': 'libgstreamer-gl1.0-0', - 'libgstpbutils-1.0.so.0': 'libgstreamer-plugins-base1.0-0', - 'libgstreamer-1.0.so.0': 'libgstreamer1.0-0', - 'libgsttag-1.0.so.0': 'libgstreamer-plugins-base1.0-0', - 'libgstvideo-1.0.so.0': 'libgstreamer-plugins-base1.0-0', - 'libgtk-3.so.0': 'libgtk-3-0', - 'libgudev-1.0.so.0': 'libgudev-1.0-0', - 'libharfbuzz-icu.so.0': 'libharfbuzz-icu0', - 'libharfbuzz.so.0': 'libharfbuzz0b', - 'libhyphen.so.0': 'libhyphen0', - 'libicui18n.so.67': 'libicu67', - 'libicuuc.so.67': 'libicu67', - 'libjpeg.so.62': 'libjpeg62-turbo', - 'liblcms2.so.2': 'liblcms2-2', - 'libmanette-0.2.so.0': 'libmanette-0.2-0', - 'libnotify.so.4': 'libnotify4', - 'libnspr4.so': 'libnspr4', - 'libnss3.so': 'libnss3', - 'libnssutil3.so': 'libnss3', - 'libOpenGL.so.0': 'libopengl0', - 'libopenjp2.so.7': 'libopenjp2-7', - 'libopus.so.0': 'libopus0', - 'libpango-1.0.so.0': 'libpango-1.0-0', - 'libpangocairo-1.0.so.0': 'libpangocairo-1.0-0', - 'libpng16.so.16': 'libpng16-16', - 'libproxy.so.1': 'libproxy1v5', - 'libsecret-1.so.0': 'libsecret-1-0', - 'libsmime3.so': 'libnss3', - 'libwayland-client.so.0': 'libwayland-client0', - 'libwayland-egl.so.1': 'libwayland-egl1', - 'libwayland-server.so.0': 'libwayland-server0', - 'libwebp.so.6': 'libwebp6', - 'libwebpdemux.so.2': 'libwebpdemux2', - 'libwoff2dec.so.1.0.2': 'libwoff1', - 'libX11-xcb.so.1': 'libx11-xcb1', - 'libX11.so.6': 'libx11-6', - 'libxcb-shm.so.0': 'libxcb-shm0', - 'libxcb.so.1': 'libxcb1', - 'libXcomposite.so.1': 'libxcomposite1', - 'libXcursor.so.1': 'libxcursor1', - 'libXdamage.so.1': 'libxdamage1', - 'libXext.so.6': 'libxext6', - 'libXfixes.so.3': 'libxfixes3', - 'libXi.so.6': 'libxi6', - 'libxkbcommon.so.0': 'libxkbcommon0', - 'libxml2.so.2': 'libxml2', - 'libXrandr.so.2': 'libxrandr2', - 'libXrender.so.1': 'libxrender1', - 'libxslt.so.1': 'libxslt1.1', - 'libXtst.so.6': 'libxtst6', - 'libatomic.so.1': 'libatomic1', - 'libevent-2.1.so.7': 'libevent-2.1-7', - } - }, 'debian12-x64': { tools: [ 'xvfb', @@ -1471,20 +1260,6 @@ deps['ubuntu26.04-arm64'] = { }, }; -deps['debian11-arm64'] = { - tools: [...deps['debian11-x64'].tools], - chromium: [...deps['debian11-x64'].chromium], - firefox: [ - ...deps['debian11-x64'].firefox, - ], - webkit: [ - ...deps['debian11-x64'].webkit, - ], - lib2package: { - ...deps['debian11-x64'].lib2package, - }, -}; - deps['debian12-arm64'] = { tools: [...deps['debian12-x64'].tools], chromium: [...deps['debian12-x64'].chromium], diff --git a/packages/playwright-core/src/server/webkit/wkPage.ts b/packages/playwright-core/src/server/webkit/wkPage.ts index f9c9ed6b907a4..ef084aed771e6 100644 --- a/packages/playwright-core/src/server/webkit/wkPage.ts +++ b/packages/playwright-core/src/server/webkit/wkPage.ts @@ -850,8 +850,7 @@ export class WKPage implements PageDelegate { private _toolbarHeight(): number { if (this._page.browserContext._browser?.options.headful) { - if (hostPlatform === 'mac10.15') - return 55; + // note: historically, value for mac10.15 was 55 if (hostPlatform === 'mac26-arm64' || hostPlatform === 'mac26') return 69; return 59; @@ -960,9 +959,9 @@ export class WKPage implements PageDelegate { frameSwapWallTime: event.timestamp // timestamp is in seconds, we need to convert to milliseconds. ? event.timestamp * 1000 - // Fallback for Debian 11 and Ubuntu 20.04 where WebKit is frozen on an older + // Fallback for Ubuntu 20.04 where WebKit is frozen on an older // version that did not send timestamp. - // TODO: remove this fallback when Debian 11 and Ubuntu 20.04 are EOL. + // TODO: remove this fallback when Ubuntu 20.04 is EOL. : Date.now(), viewportWidth: event.deviceWidth, viewportHeight: event.deviceHeight, diff --git a/packages/playwright-core/types/types.d.ts b/packages/playwright-core/types/types.d.ts index ed7f5c099bee9..e0f0b6486db84 100644 --- a/packages/playwright-core/types/types.d.ts +++ b/packages/playwright-core/types/types.d.ts @@ -3080,13 +3080,6 @@ export interface Page { * @param options */ getByRole(role: "alert"|"alertdialog"|"application"|"article"|"banner"|"blockquote"|"button"|"caption"|"cell"|"checkbox"|"code"|"columnheader"|"combobox"|"complementary"|"contentinfo"|"definition"|"deletion"|"dialog"|"directory"|"document"|"emphasis"|"feed"|"figure"|"form"|"generic"|"grid"|"gridcell"|"group"|"heading"|"img"|"insertion"|"link"|"list"|"listbox"|"listitem"|"log"|"main"|"marquee"|"math"|"meter"|"menu"|"menubar"|"menuitem"|"menuitemcheckbox"|"menuitemradio"|"navigation"|"none"|"note"|"option"|"paragraph"|"presentation"|"progressbar"|"radio"|"radiogroup"|"region"|"row"|"rowgroup"|"rowheader"|"scrollbar"|"search"|"searchbox"|"separator"|"slider"|"spinbutton"|"status"|"strong"|"subscript"|"superscript"|"switch"|"tab"|"table"|"tablist"|"tabpanel"|"term"|"textbox"|"time"|"timer"|"toolbar"|"tooltip"|"tree"|"treegrid"|"treeitem", options?: { - /** - * An attribute that is usually set by `aria-busy`. - * - * Learn more about [`aria-busy`](https://www.w3.org/TR/wai-aria-1.2/#aria-busy). - */ - busy?: boolean; - /** * An attribute that is usually set by `aria-checked` or native `` controls. * @@ -7379,13 +7372,6 @@ export interface Frame { * @param options */ getByRole(role: "alert"|"alertdialog"|"application"|"article"|"banner"|"blockquote"|"button"|"caption"|"cell"|"checkbox"|"code"|"columnheader"|"combobox"|"complementary"|"contentinfo"|"definition"|"deletion"|"dialog"|"directory"|"document"|"emphasis"|"feed"|"figure"|"form"|"generic"|"grid"|"gridcell"|"group"|"heading"|"img"|"insertion"|"link"|"list"|"listbox"|"listitem"|"log"|"main"|"marquee"|"math"|"meter"|"menu"|"menubar"|"menuitem"|"menuitemcheckbox"|"menuitemradio"|"navigation"|"none"|"note"|"option"|"paragraph"|"presentation"|"progressbar"|"radio"|"radiogroup"|"region"|"row"|"rowgroup"|"rowheader"|"scrollbar"|"search"|"searchbox"|"separator"|"slider"|"spinbutton"|"status"|"strong"|"subscript"|"superscript"|"switch"|"tab"|"table"|"tablist"|"tabpanel"|"term"|"textbox"|"time"|"timer"|"toolbar"|"tooltip"|"tree"|"treegrid"|"treeitem", options?: { - /** - * An attribute that is usually set by `aria-busy`. - * - * Learn more about [`aria-busy`](https://www.w3.org/TR/wai-aria-1.2/#aria-busy). - */ - busy?: boolean; - /** * An attribute that is usually set by `aria-checked` or native `` controls. * @@ -15262,13 +15248,6 @@ export interface Locator { * @param options */ getByRole(role: "alert"|"alertdialog"|"application"|"article"|"banner"|"blockquote"|"button"|"caption"|"cell"|"checkbox"|"code"|"columnheader"|"combobox"|"complementary"|"contentinfo"|"definition"|"deletion"|"dialog"|"directory"|"document"|"emphasis"|"feed"|"figure"|"form"|"generic"|"grid"|"gridcell"|"group"|"heading"|"img"|"insertion"|"link"|"list"|"listbox"|"listitem"|"log"|"main"|"marquee"|"math"|"meter"|"menu"|"menubar"|"menuitem"|"menuitemcheckbox"|"menuitemradio"|"navigation"|"none"|"note"|"option"|"paragraph"|"presentation"|"progressbar"|"radio"|"radiogroup"|"region"|"row"|"rowgroup"|"rowheader"|"scrollbar"|"search"|"searchbox"|"separator"|"slider"|"spinbutton"|"status"|"strong"|"subscript"|"superscript"|"switch"|"tab"|"table"|"tablist"|"tabpanel"|"term"|"textbox"|"time"|"timer"|"toolbar"|"tooltip"|"tree"|"treegrid"|"treeitem", options?: { - /** - * An attribute that is usually set by `aria-busy`. - * - * Learn more about [`aria-busy`](https://www.w3.org/TR/wai-aria-1.2/#aria-busy). - */ - busy?: boolean; - /** * An attribute that is usually set by `aria-checked` or native `` controls. * @@ -21077,13 +21056,6 @@ export interface FrameLocator { * @param options */ getByRole(role: "alert"|"alertdialog"|"application"|"article"|"banner"|"blockquote"|"button"|"caption"|"cell"|"checkbox"|"code"|"columnheader"|"combobox"|"complementary"|"contentinfo"|"definition"|"deletion"|"dialog"|"directory"|"document"|"emphasis"|"feed"|"figure"|"form"|"generic"|"grid"|"gridcell"|"group"|"heading"|"img"|"insertion"|"link"|"list"|"listbox"|"listitem"|"log"|"main"|"marquee"|"math"|"meter"|"menu"|"menubar"|"menuitem"|"menuitemcheckbox"|"menuitemradio"|"navigation"|"none"|"note"|"option"|"paragraph"|"presentation"|"progressbar"|"radio"|"radiogroup"|"region"|"row"|"rowgroup"|"rowheader"|"scrollbar"|"search"|"searchbox"|"separator"|"slider"|"spinbutton"|"status"|"strong"|"subscript"|"superscript"|"switch"|"tab"|"table"|"tablist"|"tabpanel"|"term"|"textbox"|"time"|"timer"|"toolbar"|"tooltip"|"tree"|"treegrid"|"treeitem", options?: { - /** - * An attribute that is usually set by `aria-busy`. - * - * Learn more about [`aria-busy`](https://www.w3.org/TR/wai-aria-1.2/#aria-busy). - */ - busy?: boolean; - /** * An attribute that is usually set by `aria-checked` or native `` controls. * diff --git a/packages/playwright/src/common/config.ts b/packages/playwright/src/common/config.ts index fc4e378625ea5..0287e87d3ee4f 100644 --- a/packages/playwright/src/common/config.ts +++ b/packages/playwright/src/common/config.ts @@ -48,7 +48,7 @@ export class FullConfigInternal { readonly projects: FullProjectInternal[] = []; readonly singleTSConfigPath?: string; readonly captureGitInfo: Config['captureGitInfo']; - readonly retryStrategy: 'immediate' | 'deferred'; + readonly retryStrategy: 'immediate' | 'isolated'; defineConfigWasUsed = false; globalSetups: string[] = []; diff --git a/packages/playwright/src/common/configLoader.ts b/packages/playwright/src/common/configLoader.ts index 9beee268caa5c..f9ec6545177c5 100644 --- a/packages/playwright/src/common/configLoader.ts +++ b/packages/playwright/src/common/configLoader.ts @@ -244,8 +244,8 @@ function validateConfig(file: string, config: Config) { } if ('retryStrategy' in config && config.retryStrategy !== undefined) { - if (typeof config.retryStrategy !== 'string' || !['immediate', 'deferred'].includes(config.retryStrategy)) - throw errorWithFile(file, `config.retryStrategy must be one of "immediate" or "deferred"`); + if (typeof config.retryStrategy !== 'string' || !['immediate', 'isolated'].includes(config.retryStrategy)) + throw errorWithFile(file, `config.retryStrategy must be one of "immediate" or "isolated"`); } if ('tsconfig' in config && config.tsconfig !== undefined) { diff --git a/packages/playwright/src/common/test.ts b/packages/playwright/src/common/test.ts index 598a041d8a81e..60d1fc2e2156f 100644 --- a/packages/playwright/src/common/test.ts +++ b/packages/playwright/src/common/test.ts @@ -16,7 +16,6 @@ import { rootTestType } from './testType'; import { computeTestCaseOutcome } from '../isomorphic/teleReceiver'; -import { wrapFunctionWithLocation } from '../transform/transform'; import type { FixturesWithLocation, FullProjectInternal } from './config'; import type { FixturePool } from './fixtures'; import type { TestTypeImpl } from './testType'; @@ -58,19 +57,11 @@ export class Suite extends Base { _parallelMode: 'none' | 'default' | 'serial' | 'parallel' = 'none'; _fullProject: FullProjectInternal | undefined; _fileId: string | undefined; - _preprocessMode: 'editable' | 'readonly' | undefined = undefined; readonly _type: 'root' | 'project' | 'file' | 'describe'; - skip: (reason?: string) => void; - fixme: (reason?: string) => void; - fail: (reason?: string) => void; - constructor(title: string, type: 'root' | 'project' | 'file' | 'describe') { super(title); this._type = type; - this.skip = wrapFunctionWithLocation((location, reason?: string) => this._modifier('skip', location, reason)); - this.fixme = wrapFunctionWithLocation((location, reason?: string) => this._modifier('fixme', location, reason)); - this.fail = wrapFunctionWithLocation((location, reason?: string) => this._modifier('fail', location, reason)); } get type(): 'root' | 'project' | 'file' | 'describe' { @@ -269,30 +260,6 @@ export class Suite extends Base { return this._fullProject?.project || this.parent?.project(); } - private _modifier(type: 'skip' | 'fixme' | 'fail', location: Location, reason: string | undefined): void { - const mode = this._resolvePreprocessMode(); - if (!mode) - throw new Error(`Suite.${type}() can only be called from Reporter.preprocessSuite().`); - if (mode === 'readonly') - throw new Error(`Suite.${type}() cannot be called on a setup or teardown project; these always run in full.`); - for (const test of this.allTests()) - test._applyPlanAnnotation({ type, description: reason, location }); - } - - exclude(): void { - const mode = this._resolvePreprocessMode(); - if (!mode) - throw new Error(`Suite.exclude() can only be called from Reporter.preprocessSuite().`); - if (!this.parent) - throw new Error(`Suite.exclude() cannot be called on the root suite.`); - if (mode === 'readonly') - throw new Error(`Suite.exclude() cannot be called on a setup or teardown project; these always run in full.`); - this.parent._detach(this); - } - - _resolvePreprocessMode(): 'editable' | 'readonly' | undefined { - return this._preprocessMode ?? this.parent?._resolvePreprocessMode(); - } } export class TestCase extends Base implements reporterTypes.TestCase { @@ -318,18 +285,11 @@ export class TestCase extends Base implements reporterTypes.TestCase { _tags: string[] = []; _planAnnotations: TestAnnotation[] = []; - skip: (reason?: string) => void; - fixme: (reason?: string) => void; - fail: (reason?: string) => void; - constructor(title: string, fn: Function, testType: TestTypeImpl, location: Location) { super(title); this.fn = fn; this._testType = testType; this.location = location; - this.skip = wrapFunctionWithLocation((location, reason?: string) => this._modifier('skip', location, reason)); - this.fixme = wrapFunctionWithLocation((location, reason?: string) => this._modifier('fixme', location, reason)); - this.fail = wrapFunctionWithLocation((location, reason?: string) => this._modifier('fail', location, reason)); } titlePath(): string[] { @@ -358,15 +318,6 @@ export class TestCase extends Base implements reporterTypes.TestCase { ]; } - private _modifier(type: 'skip' | 'fixme' | 'fail', location: Location, reason: string | undefined): void { - const mode = this.parent._resolvePreprocessMode(); - if (!mode) - throw new Error(`TestCase.${type}() can only be called from Reporter.preprocessSuite().`); - if (mode === 'readonly') - throw new Error(`TestCase.${type}() cannot be called on a setup or teardown project test; these always run in full.`); - this._applyPlanAnnotation({ type, description: reason, location }); - } - _applyPlanAnnotation(annotation: TestAnnotation): void { this.annotations.push(annotation); this._planAnnotations.push(annotation); @@ -376,15 +327,6 @@ export class TestCase extends Base implements reporterTypes.TestCase { this.expectedStatus = 'failed'; } - exclude(): void { - const mode = this.parent._resolvePreprocessMode(); - if (!mode) - throw new Error(`TestCase.exclude() can only be called from Reporter.preprocessSuite().`); - if (mode === 'readonly') - throw new Error(`TestCase.exclude() cannot be called on a setup or teardown project test; these always run in full.`); - this.parent._detach(this); - } - _serialize(): any { return { kind: 'test', diff --git a/packages/playwright/src/reporters/html.ts b/packages/playwright/src/reporters/html.ts index 8bc5cdc8586df..8fe90bcbbf822 100644 --- a/packages/playwright/src/reporters/html.ts +++ b/packages/playwright/src/reporters/html.ts @@ -157,11 +157,13 @@ class HtmlReporter implements ReporterV2 { const noSnippets = parseBooleanEnvVar('PLAYWRIGHT_HTML_NO_SNIPPETS') ?? this._options.noSnippets; const noCopyPrompt = parseBooleanEnvVar('PLAYWRIGHT_HTML_NO_COPY_PROMPT') ?? this._options.noCopyPrompt; const doNotInlineAssets = parseBooleanEnvVar('PLAYWRIGHT_HTML_DO_NOT_INLINE_ASSETS') ?? this._options.doNotInlineAssets ?? false; + const mergeFiles = parseBooleanEnvVar('PLAYWRIGHT_HTML_MERGE_FILES') ?? this._options.mergeFiles; const builder = new HtmlBuilder(yazl, this.config, this._outputFolder, this._attachmentsBaseURL, doNotInlineAssets, { title: process.env.PLAYWRIGHT_HTML_TITLE || this._options.title, noSnippets, noCopyPrompt, + mergeFiles, }); this._buildResult = await builder.build(this.config.metadata, projectSuites, result, this._topLevelErrors, this._machines); } diff --git a/packages/playwright/src/reporters/internalReporter.ts b/packages/playwright/src/reporters/internalReporter.ts index f4f092cd5d9ea..26e9ba42dcb34 100644 --- a/packages/playwright/src/reporters/internalReporter.ts +++ b/packages/playwright/src/reporters/internalReporter.ts @@ -24,10 +24,9 @@ import { test as testNs } from '../common'; import * as babel from '../transform/babelBundle'; import { wrapReporterAsV2 } from './reporterV2'; -import type { AnyReporter, ReporterV2 } from './reporterV2'; +import type { AnyReporter, ReporterPreprocessParams, ReporterV2 } from './reporterV2'; import type { FullConfig, FullResult, TestCase, TestError, TestResult, TestStep, WorkerInfo } from '../../types/testReporter'; - export class InternalReporter implements ReporterV2 { private _reporter: Multiplexer; private _didBegin = false; @@ -54,8 +53,8 @@ export class InternalReporter implements ReporterV2 { this._reporter.onConfigure?.(config); } - async preprocessSuite(config: FullConfig, suite: testNs.Suite) { - return await this._reporter.preprocessSuite?.(config, suite); + async preprocess(params: ReporterPreprocessParams) { + await this._reporter.preprocess(params); } onBegin(suite: testNs.Suite) { diff --git a/packages/playwright/src/reporters/multiplexer.ts b/packages/playwright/src/reporters/multiplexer.ts index 9a57c13ed72ca..2a370e8fd8d1c 100644 --- a/packages/playwright/src/reporters/multiplexer.ts +++ b/packages/playwright/src/reporters/multiplexer.ts @@ -16,7 +16,7 @@ import { serializeError } from '../util'; -import type { ReportConfigureParams, ReportEndParams, ReporterV2 } from './reporterV2'; +import type { ReportConfigureParams, ReportEndParams, ReporterPreprocessParams, ReporterV2 } from './reporterV2'; import type { FullConfig, FullResult, TestCase, TestError, TestResult, TestStep, WorkerInfo } from '../../types/testReporter'; import type { test } from '../common'; @@ -41,20 +41,13 @@ export class Multiplexer implements ReporterV2 { this._wrap(() => reporter.onConfigure?.(config)); } - async preprocessSuite(config: FullConfig, suite: test.Suite) { - // Unlike other reporter callbacks, `preprocessSuite` errors are NOT swallowed — - // they propagate so the run aborts before onBegin. Reporters use preprocessSuite + async preprocess(params: ReporterPreprocessParams) { + // Unlike other reporter callbacks, `preprocess` errors are NOT swallowed — + // they propagate so the run aborts before onBegin. Reporters use preprocess // to mutate the corpus; silently dropping a planning error would let // an inconsistent (partial-mutation) state reach the workers. - const shardingReporters: ReporterV2[] = []; - for (const reporter of this._reporters) { - const result = await reporter.preprocessSuite?.(config, suite); - if (result?.implementsSharding) - shardingReporters.push(reporter); - } - if (shardingReporters.length > 1) - throw new Error(`Multiple reporters declare 'implementsSharding': ${shardingReporters.map(r => r.constructor?.name ?? 'reporter').join(', ')}. Only one reporter may handle sharding.`); - return { implementsSharding: shardingReporters.length > 0 }; + for (const reporter of this._reporters) + await reporter.preprocess?.(params); } onBegin(suite: test.Suite) { diff --git a/packages/playwright/src/reporters/reporterV2.ts b/packages/playwright/src/reporters/reporterV2.ts index 3d5a65a53d46a..131e2475a5438 100644 --- a/packages/playwright/src/reporters/reporterV2.ts +++ b/packages/playwright/src/reporters/reporterV2.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import type { FullConfig, FullResult, Reporter, Suite, TestCase, TestError, TestResult, TestStep, WorkerInfo } from '../../types/testReporter'; +import type { FullConfig, FullResult, Reporter, Suite, TestCase, TestError, TestResult, TestRun, TestStep, WorkerInfo } from '../../types/testReporter'; export interface ReportConfigureParams { config: FullConfig; @@ -26,9 +26,15 @@ export interface ReportEndParams { result: FullResult; } +export type ReporterPreprocessParams = { + config: FullConfig; + suite: Suite; + testRun: TestRun; +}; + export interface ReporterV2 { onConfigure?(config: FullConfig): void; - preprocessSuite?(config: FullConfig, suite: Suite): { implementsSharding?: boolean } | Promise<{ implementsSharding?: boolean } | undefined | void> | void; + preprocess?(params: ReporterPreprocessParams): Promise | void; onBegin?(suite: Suite): void; onTestBegin?(test: TestCase, result: TestResult): void; onStdOut?(chunk: string | Buffer, test?: TestCase, result?: TestResult): void; @@ -80,8 +86,8 @@ class ReporterV2Wrapper implements ReporterV2 { this._config = config; } - async preprocessSuite(config: FullConfig, suite: Suite) { - return await this._reporter.preprocessSuite?.(config, suite); + preprocess(params: ReporterPreprocessParams) { + return this._reporter.preprocess?.(params); } onBegin(suite: Suite) { diff --git a/packages/playwright/src/runner/dispatcher.ts b/packages/playwright/src/runner/dispatcher.ts index aa30f5fb85579..87261df5dfd5c 100644 --- a/packages/playwright/src/runner/dispatcher.ts +++ b/packages/playwright/src/runner/dispatcher.ts @@ -36,6 +36,7 @@ export class Dispatcher { // Worker slot is claimed when it has jobDispatcher assigned. private _workerSlots: { worker?: WorkerHost, jobDispatcher?: JobDispatcher }[] = []; private _queue: TestGroup[] = []; + private _isolatedJobs = new Set(); private _workerLimitPerProjectId = new Map(); private _queuedOrRunningHashCount = new Map(); private _finished = new ManualPromise(); @@ -58,6 +59,9 @@ export class Dispatcher { // Always pick the first job that can be run while respecting the project worker limit. for (let index = 0; index < this._queue.length; index++) { const job = this._queue[index]; + // Isolated retries only run one at a time, after all other jobs have finished. + if (this._isolatedJobs.has(job) && this._workerSlots.some(w => !!w.jobDispatcher)) + continue; const projectIdWorkerLimit = this._workerLimitPerProjectId.get(job.projectId); if (!projectIdWorkerLimit) return index; @@ -150,13 +154,17 @@ export class Dispatcher { else if (this._isWorkerRedundant(worker)) void worker.stop(); - // 5. Possibly queue a new job with leftover tests and/or retries. - if (!this._isStopped && result.newJob) { - if (this._testRun.config.retryStrategy === 'deferred') - this._queue.push(result.newJob); - else - this._queue.unshift(result.newJob); - this._updateCounterForWorkerHash(result.newJob.workerHash, +1); + // 5. Possibly queue new jobs with leftover tests and/or retries. + if (!this._isStopped) { + if (result.remainingJob) { + this._queue.unshift(result.remainingJob); + this._updateCounterForWorkerHash(result.remainingJob.workerHash, +1); + } + if (result.isolatedRetriesJob) { + this._isolatedJobs.add(result.isolatedRetriesJob); + this._queue.push(result.isolatedRetriesJob); + this._updateCounterForWorkerHash(result.isolatedRetriesJob.workerHash, +1); + } } } @@ -280,7 +288,7 @@ export class Dispatcher { } class JobDispatcher { - jobResult = new ManualPromise<{ newJob?: TestGroup, didFail: boolean }>(); + jobResult = new ManualPromise<{ remainingJob?: TestGroup, isolatedRetriesJob?: TestGroup, didFail: boolean }>(); readonly job: TestGroup; private _testRun: TestRun; @@ -537,14 +545,21 @@ class JobDispatcher { } const remaining = [...this._remainingByTestId.values()]; + const isolatedRetries: testNs.TestCase[] = []; for (const test of retryCandidates) { - if (test.results.length < test.retries + 1) - remaining.push(test); + if (test.results.length < test.retries + 1) { + // Immediate retries run together with the remaining tests, in a single job. + if (this._testRun.config.retryStrategy === 'immediate') + remaining.push(test); + else + isolatedRetries.push(test); + } } - // This job is over, we will schedule another one. - const newJob = remaining.length ? { ...this.job, tests: remaining } : undefined; - this._finished({ didFail: true, newJob }); + // This job is over, we will schedule new jobs for the remaining tests and isolated retries. + const remainingJob = remaining.length ? { ...this.job, tests: remaining } : undefined; + const isolatedRetriesJob = isolatedRetries.length ? { ...this.job, tests: isolatedRetries } : undefined; + this._finished({ didFail: true, remainingJob, isolatedRetriesJob }); } onExit(data: ProcessExitData) { @@ -554,7 +569,7 @@ class JobDispatcher { this._onDone({ skipTestsDueToSetupFailure: [], fatalErrors: [], unexpectedExitError }); } - private _finished(result: { newJob?: TestGroup, didFail: boolean }) { + private _finished(result: { remainingJob?: TestGroup, isolatedRetriesJob?: TestGroup, didFail: boolean }) { eventsHelper.removeEventListeners(this._listeners); this.jobResult.resolve(result); } diff --git a/packages/playwright/src/runner/loadUtils.ts b/packages/playwright/src/runner/loadUtils.ts index 9878b14702c8b..80553cb8e232e 100644 --- a/packages/playwright/src/runner/loadUtils.ts +++ b/packages/playwright/src/runner/loadUtils.ts @@ -23,7 +23,8 @@ import { toPosixPath } from '@utils/fileUtils'; import { InProcessLoaderHost, OutOfProcessLoaderHost } from './loaderHost'; import { createTitleMatcher, errorWithFile, parseLocationArg } from '../util'; import { buildProjectsClosure, collectFilesForProject } from './projectUtils'; -import { createTestGroups, filterForShard } from './testGroups'; +import { ReporterTestRunImpl } from './reporterTestRun'; +import { createTestGroups, filterForShard } from './testGroups'; import { cc, config as commonConfig, FullConfigInternal, suiteUtils, test as testNs, transform } from '../common'; import type { RawSourceMap } from 'source-map'; @@ -32,7 +33,6 @@ import type { TestGroup } from './testGroups'; import type { FullConfig, Reporter, TestError } from '../../types/testReporter'; import type { Matcher, TestCaseFilter } from '../util'; - export async function collectProjectsAndTestFiles(testRun: TestRun, doNotRunTestsOutsideProjectFilter: boolean) { const fsCache = new Map(); const sourceMapCache = new Map(); @@ -171,26 +171,24 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho if (type !== 'dependency') continue; const dependencySuite = buildProjectSuite(project, projectSuites.get(project)!); - dependencySuite._preprocessMode = 'readonly'; dependencySuites.set(project, dependencySuite); rootSuite._prependSuite(dependencySuite); } - rootSuite._preprocessMode = 'editable'; - let preprocessResult: Awaited>; - try { - preprocessResult = await testRun.reporter.preprocessSuite(config.config, rootSuite); - } finally { - // Continue the existing sharding and filtering pipeline with top-level projects only. - rootSuite._preprocessMode = undefined; - for (const dependencySuite of dependencySuites.values()) { - dependencySuite._preprocessMode = undefined; - rootSuite._detach(dependencySuite); - } - } + const reporterTestRun = new ReporterTestRunImpl(rootSuite, new Set(dependencySuites.values())); + await testRun.reporter.preprocess({ + config: config.config, + suite: rootSuite, + testRun: reporterTestRun, + }); + reporterTestRun.close(); + + // Continue sharding and filtering pipeline with top-level projects only. + for (const dependencySuite of dependencySuites.values()) + rootSuite._detach(dependencySuite); // Shard only the top-level projects. - if (config.config.shard && !preprocessResult?.implementsSharding) { + if (config.config.shard && !reporterTestRun.shouldSkipSharding()) { // Create test groups for top-level projects. const testGroups: TestGroup[] = []; for (const projectSuite of rootSuite.suites) { @@ -222,12 +220,15 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho const finalProjectClosure = buildProjectsClosure(rootSuite.suites.map(suite => suite._fullProject!)); for (const [project, type] of finalProjectClosure) { if (type === 'dependency') - rootSuite._prependSuite(dependencySuites.get(project)!); + rootSuite._addSuite(dependencySuites.get(project)!); else topLevelProjects.push(project); } } + // Keep project suites in the order they are declared in the config. + rootSuite._entries.sort((a, b) => config.projects.indexOf((a as testNs.Suite)._fullProject!) - config.projects.indexOf((b as testNs.Suite)._fullProject!)); + testRun.rootSuite = rootSuite; testRun.topLevelProjects = topLevelProjects; } diff --git a/packages/playwright/src/runner/reporterTestRun.ts b/packages/playwright/src/runner/reporterTestRun.ts new file mode 100644 index 0000000000000..c0e0c11ecaebf --- /dev/null +++ b/packages/playwright/src/runner/reporterTestRun.ts @@ -0,0 +1,83 @@ +/** + * Copyright Microsoft Corporation. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { test as testNs, transform } from '../common'; + +import type { Location, Suite as ReporterSuite, TestCase as ReporterTestCase, TestRun as ReporterTestRun } from '../../types/testReporter'; + +type ReporterTestRunTarget = ReporterSuite | ReporterTestCase; + +export class ReporterTestRunImpl implements ReporterTestRun { + private _active = true; + private _skipSharding = false; + + constructor(private _rootSuite: testNs.Suite, private _readonlyProjectSuites: Set) {} + + close() { + this._active = false; + } + + shouldSkipSharding() { + return this._skipSharding; + } + + skipSharding() { + if (!this._active) + throw new Error(`TestRun.skipSharding() can only be called from Reporter.preprocess().`); + if (this._skipSharding) + throw new Error(`Multiple reporters called 'skipSharding'. Only one reporter may handle sharding.`); + this._skipSharding = true; + } + + skip = transform.wrapFunctionWithLocation((location: Location, target: ReporterTestRunTarget, reason?: string) => this._modifier('skip', location, target, reason)); + fixme = transform.wrapFunctionWithLocation((location: Location, target: ReporterTestRunTarget, reason?: string) => this._modifier('fixme', location, target, reason)); + fail = transform.wrapFunctionWithLocation((location: Location, target: ReporterTestRunTarget, reason?: string) => this._modifier('fail', location, target, reason)); + + exclude(target: ReporterTestRunTarget) { + if (!this._active) + throw new Error(`TestRun.exclude() can only be called from Reporter.preprocess().`); + const internalTarget = target as testNs.Suite | testNs.TestCase; + this._checkTarget('exclude', internalTarget); + internalTarget.parent!._detach(internalTarget); + } + + private _modifier(type: 'skip' | 'fixme' | 'fail', location: Location, _target: ReporterTestRunTarget, reason: string | undefined) { + if (!this._active) + throw new Error(`TestRun.${type}() can only be called from Reporter.preprocess().`); + const target = _target as testNs.Suite | testNs.TestCase; + this._checkTarget(type, target); + const annotation = { type, description: reason, location }; + if (target instanceof testNs.Suite) { + for (const test of target.allTests()) + test._applyPlanAnnotation(annotation); + } else { + target._applyPlanAnnotation(annotation); + } + } + + private _checkTarget(method: string, target: testNs.Suite | testNs.TestCase) { + if (target === this._rootSuite) + throw new Error(`TestRun.${method}() cannot be called on the root suite.`); + let suite: testNs.Suite | undefined = target instanceof testNs.Suite ? target : target.parent; + while (suite) { + if (this._readonlyProjectSuites.has(suite)) { + const targetType = target instanceof testNs.TestCase ? ' test' : ''; + throw new Error(`TestRun.${method}() cannot be called on a setup or teardown project${targetType}; these always run in full.`); + } + suite = suite.parent; + } + } +} diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts index 093da901d0804..d318379841738 100644 --- a/packages/playwright/types/test.d.ts +++ b/packages/playwright/types/test.d.ts @@ -32,6 +32,7 @@ export type HtmlReporterOptions = { noSnippets?: boolean; noCopyPrompt?: boolean; doNotInlineAssets?: boolean; + mergeFiles?: boolean; }; export type ReporterDescription = Readonly< @@ -1644,8 +1645,9 @@ interface TestConfig { * Controls when failed tests are retried. Defaults to `'immediate'`. * - `'immediate'` - A failed test is retried as soon as a worker is available, interleaved with the rest of the * run. This is the default. - * - `'deferred'` - Retries are run only after all tests have had their first attempt, in parallel up to the - * configured number of [workers](#test-config-workers). + * - `'isolated'` - Retries are run at the end, after all other tests have finished, one by one in a single worker. + * This minimizes the interference between retried tests and the rest of the suite, at the expense of the total + * run time. * * Learn more about [test retries](https://playwright.dev/docs/test-retries#retries). * @@ -1657,12 +1659,12 @@ interface TestConfig { * * export default defineConfig({ * retries: 2, - * retryStrategy: 'deferred', + * retryStrategy: 'isolated', * }); * ``` * */ - retryStrategy?: "immediate"|"deferred"; + retryStrategy?: "immediate"|"isolated"; /** * Shard tests and execute only the selected shard. Specify in the one-based form like `{ total: 5, current: 2 }`. diff --git a/packages/playwright/types/testReporter.d.ts b/packages/playwright/types/testReporter.d.ts index f7809c29c65fe..550b27c86090a 100644 --- a/packages/playwright/types/testReporter.d.ts +++ b/packages/playwright/types/testReporter.d.ts @@ -145,22 +145,6 @@ export interface FullResult { * [reporter.onBegin(config, suite)](https://playwright.dev/docs/api/class-reporter#reporter-on-begin). */ export interface Reporter { - /** - * Called after the configuration has been resolved and before - * [reporter.onBegin(config, suite)](https://playwright.dev/docs/api/class-reporter#reporter-on-begin). Allows a - * reporter to mark individual tests as skipped, excluded, fixed or failing. - * @param config Resolved configuration. - * @param suite The root suite that contains the projects, files and test cases that will run. - * - * The suite reflects `--project`, `--grep`/`--grep-invert` and `.only` filtering, so it only contains tests that - * match the current invocation. Setup and dependency projects are readonly and cannot be excluded from here. - * - * The suite ignores the `--shard` argument: it always contains the full, un-sharded corpus. Playwright applies its - * built-in sharding after - * [reporter.preprocessSuite(config, suite)](https://playwright.dev/docs/api/class-reporter#reporter-preprocess-suite) - * returns, unless the returned `implementsSharding` is `true`. - */ - preprocessSuite?(config: FullConfig, suite: Suite): Promise<{ implementsSharding?: boolean } | undefined | void> | { implementsSharding?: boolean } | void; /** * Called after all tests have been run, or testing has been interrupted. Note that this method may return a [Promise] * and Playwright Test will await it. Reporter is allowed to override the status and hence affect the exit code of the @@ -243,6 +227,36 @@ export interface Reporter { */ onTestEnd?(test: TestCase, result: TestResult): void; + /** + * Called after the configuration has been resolved and before + * [reporter.onBegin(config, suite)](https://playwright.dev/docs/api/class-reporter#reporter-on-begin). Allows a + * reporter to mark individual tests as skipped, excluded, fixed or failing. + * @param params The suite reflects `--project`, `--grep`/`--grep-invert` and `.only` filtering, so it only contains tests that + * match the current invocation. Setup and dependency projects are readonly and cannot be changed through + * [TestRun](https://playwright.dev/docs/api/class-testrun). + * + * The suite ignores the `--shard` argument: it always contains the full, un-sharded corpus. Playwright applies its + * built-in sharding after + * [reporter.preprocess(params)](https://playwright.dev/docs/api/class-reporter#reporter-preprocess) returns, unless + * the reporter calls [testRun.skipSharding()](https://playwright.dev/docs/api/class-testrun#test-run-skip-sharding). + */ + preprocess?(params: { + /** + * Resolved configuration. + */ + config: FullConfig; + + /** + * The root suite that contains the projects, files and test cases that will run. + */ + suite: Suite; + + /** + * Control which tests will run and their expected status. + */ + testRun: TestRun; + }): Promise; + /** * Whether this reporter uses stdio for reporting. When it does not, Playwright Test could add some output to enhance * user experience. If your reporter does not print to the terminal, it is strongly recommended to return `false`. @@ -384,45 +398,11 @@ export interface Suite { */ entries(): Array; - /** - * Must be called from inside - * [reporter.preprocessSuite(config, suite)](https://playwright.dev/docs/api/class-reporter#reporter-preprocess-suite), - * exclude this suite from the run. Excluded tests do not appear in the report and their body is not executed. - */ - exclude(): void; - - /** - * Must be called from inside - * [reporter.preprocessSuite(config, suite)](https://playwright.dev/docs/api/class-reporter#reporter-preprocess-suite). - * Mark every [TestCase](https://playwright.dev/docs/api/class-testcase) of this suite as expected-to-fail, see - * [testCase.fail([reason])](https://playwright.dev/docs/api/class-testcase#test-case-fail). - * @param reason Optional explanation surfaced as the annotation description. - */ - fail(reason?: string): void; - - /** - * Must be called from inside - * [reporter.preprocessSuite(config, suite)](https://playwright.dev/docs/api/class-reporter#reporter-preprocess-suite). - * Mark every [TestCase](https://playwright.dev/docs/api/class-testcase) of this suite as fixme, see - * [testCase.fixme([reason])](https://playwright.dev/docs/api/class-testcase#test-case-fixme). - * @param reason Optional explanation surfaced as the annotation description. - */ - fixme(reason?: string): void; - /** * Configuration of the project this suite belongs to, or [void] for the root suite. */ project(): FullProject|undefined; - /** - * Must be called from inside - * [reporter.preprocessSuite(config, suite)](https://playwright.dev/docs/api/class-reporter#reporter-preprocess-suite). - * Mark every [TestCase](https://playwright.dev/docs/api/class-testcase) of this suite as skipped, see - * [testCase.skip([reason])](https://playwright.dev/docs/api/class-testcase#test-case-skip). - * @param reason Optional explanation surfaced as the annotation description. - */ - skip(reason?: string): void; - /** * Returns a list of titles from the root down to this suite. */ @@ -477,31 +457,6 @@ export interface Suite { * projects' suites. */ export interface TestCase { - /** - * Must be called from inside - * [reporter.preprocessSuite(config, suite)](https://playwright.dev/docs/api/class-reporter#reporter-preprocess-suite), - * exclude this test from the run. Excluded tests do not appear in the report and their body is not executed. - */ - exclude(): void; - - /** - * Must be called from inside - * [reporter.preprocessSuite(config, suite)](https://playwright.dev/docs/api/class-reporter#reporter-preprocess-suite), - * mark this test as "should fail". Playwright runs the test and ensures it is actually failing, useful for - * documenting broken functionality until it is fixed. - * @param reason Optional explanation surfaced as the annotation description. - */ - fail(reason?: string): void; - - /** - * Must be called from inside - * [reporter.preprocessSuite(config, suite)](https://playwright.dev/docs/api/class-reporter#reporter-preprocess-suite), - * mark this test as fixme. The test body is not executed and the test is reported as skipped, with the intention to - * fix it. - * @param reason Optional explanation surfaced as the annotation description. - */ - fixme(reason?: string): void; - /** * Whether the test is considered running fine. Non-ok tests fail the test run with non-zero exit code. */ @@ -515,14 +470,6 @@ export interface TestCase { */ outcome(): "skipped"|"expected"|"unexpected"|"flaky"; - /** - * Must be called from inside - * [reporter.preprocessSuite(config, suite)](https://playwright.dev/docs/api/class-reporter#reporter-preprocess-suite), - * skip this test. The test body is not executed and the test is reported as skipped. - * @param reason Optional explanation surfaced as the annotation description. - */ - skip(reason?: string): void; - /** * Returns a list of titles from the root down to this test. */ @@ -802,7 +749,52 @@ export interface TestResult { } /** - * Represents a step in the [TestRun]. + * Controls which tests will run and their expected status. A [TestRun](https://playwright.dev/docs/api/class-testrun) + * is available during + * [reporter.preprocess(params)](https://playwright.dev/docs/api/class-reporter#reporter-preprocess). Setup and + * teardown projects cannot be changed. + */ +export interface TestRun { + /** + * Excludes a test or suite from the run. Excluded tests do not appear in the report and their bodies are not + * executed. + * @param test Test or suite to exclude. The root suite cannot be excluded. + */ + exclude(test: TestCase|Suite): void; + + /** + * Marks a test or every test in a suite as "should fail". Playwright runs the tests and ensures they are actually + * failing, useful for documenting broken functionality until it is fixed. + * @param test Test or suite to mark as expected-to-fail. + * @param reason Optional explanation surfaced as the annotation description. + */ + fail(test: TestCase|Suite, reason?: string): void; + + /** + * Marks a test or every test in a suite as fixme. The test bodies are not executed and the tests are reported as + * skipped, with the intention to fix them. + * @param test Test or suite to mark as fixme. + * @param reason Optional explanation surfaced as the annotation description. + */ + fixme(test: TestCase|Suite, reason?: string): void; + + /** + * Skips a test or every test in a suite. The test bodies are not executed and the tests are reported as skipped. + * @param test Test or suite to skip. + * @param reason Optional explanation surfaced as the annotation description. + */ + skip(test: TestCase|Suite, reason?: string): void; + + /** + * Disables Playwright's built-in shard filter for this run, leaving sharding to the reporter. Reporters typically + * implement their own sharding by calling + * [testRun.exclude(test)](https://playwright.dev/docs/api/class-testrun#test-run-exclude) on out-of-shard tests. + */ + skipSharding(): void; +} + +/** + * Represents a step in a [TestResult](https://playwright.dev/docs/api/class-testresult). */ export interface TestStep { /** diff --git a/packages/utils/hostPlatform.ts b/packages/utils/hostPlatform.ts index 46b639d9be5bb..82974b982c7c6 100644 --- a/packages/utils/hostPlatform.ts +++ b/packages/utils/hostPlatform.ts @@ -48,29 +48,28 @@ function calculatePlatform(): { hostPlatform: HostPlatform, isOfficiallySupporte const platform = os.platform(); if (platform === 'darwin') { const ver = os.release().split('.').map((a: string) => parseInt(a, 10)); - let macVersion = ''; + let macVersion = 0; if (ver[0] < 18) { // Everything before 10.14 is considered 10.13. - macVersion = 'mac10.13'; + macVersion = 10.13; } else if (ver[0] === 18) { - macVersion = 'mac10.14'; + macVersion = 10.14; } else if (ver[0] === 19) { - macVersion = 'mac10.15'; + macVersion = 10.15; } else if (ver[0] < 25) { - // Darwin 20..24 → macOS 11..15 (BigSur..Sequoia). - macVersion = 'mac' + (ver[0] - 9); - // BigSur is the first version that might run on Apple Silicon. - if (os.cpus().some(cpu => cpu.model.includes('Apple'))) - macVersion += '-arm64'; + macVersion = 11 + (ver[0] - 20); // Darwin 20..24 → macOS 11..15 (BigSur..Sequoia) } else { // Apple jumped from macOS 15 (Sequoia) to macOS 26 (Tahoe), so Darwin 25 = macOS 26. // Best-effort support for MacOS beta versions. const LAST_STABLE_MACOS_MAJOR_VERSION = 26; - macVersion = 'mac' + Math.min(ver[0] + 1, LAST_STABLE_MACOS_MAJOR_VERSION); - if (os.cpus().some(cpu => cpu.model.includes('Apple'))) - macVersion += '-arm64'; + macVersion = Math.min(ver[0] + 1, LAST_STABLE_MACOS_MAJOR_VERSION); + } + let macPlatform = 'mac' + macVersion; + if (macVersion >= 11 && os.cpus().some(cpu => cpu.model.includes('Apple'))) { + // BigSur is the first version that might run on Apple Silicon. + macPlatform += '-arm64'; } - return { hostPlatform: macVersion as HostPlatform, isOfficiallySupportedPlatform: true }; + return { hostPlatform: macPlatform as HostPlatform, isOfficiallySupportedPlatform: macVersion >= 14 }; } if (platform === 'linux') { if (!['x64', 'arm64'].includes(os.arch())) @@ -108,9 +107,9 @@ function calculatePlatform(): { hostPlatform: HostPlatform, isOfficiallySupporte return { hostPlatform: ('ubuntu24.04' + archSuffix) as HostPlatform, isOfficiallySupportedPlatform: false }; } if (distroInfo?.id === 'debian' || distroInfo?.id === 'raspbian') { - const isOfficiallySupportedPlatform = distroInfo?.id === 'debian'; if (distroInfo?.version === '11') - return { hostPlatform: ('debian11' + archSuffix) as HostPlatform, isOfficiallySupportedPlatform }; + return { hostPlatform: ('debian11' + archSuffix) as HostPlatform, isOfficiallySupportedPlatform: false }; + const isOfficiallySupportedPlatform = distroInfo?.id === 'debian'; if (distroInfo?.version === '12') return { hostPlatform: ('debian12' + archSuffix) as HostPlatform, isOfficiallySupportedPlatform }; if (distroInfo?.version === '13') diff --git a/tests/bidi/expectationReporter.ts b/tests/bidi/expectationReporter.ts index d58c6be80d5df..734095aa9c34f 100644 --- a/tests/bidi/expectationReporter.ts +++ b/tests/bidi/expectationReporter.ts @@ -35,7 +35,7 @@ class ExpectationReporter implements Reporter { this._options = options; } - async preprocessSuite(config: FullConfig, suite: Suite) { + async preprocess({ suite, testRun }: Parameters>[0]) { if (!process.env.PWTEST_USE_BIDI_EXPECTATIONS) return; if (this._options.rebase) @@ -45,7 +45,7 @@ class ExpectationReporter implements Reporter { for (const test of project.allTests()) { const expectation = expectations.get(expectationKey(test)); if (expectation && ['flaky', 'fail', 'timeout'].includes(expectation)) - test.fixme(`marked as ${expectation} in bidi expectations`); + testRun.fixme(test, `marked as ${expectation} in bidi expectations`); } } } diff --git a/tests/library/locator-generator.spec.ts b/tests/library/locator-generator.spec.ts index 28a96e633ba0c..fa9a863592e9c 100644 --- a/tests/library/locator-generator.spec.ts +++ b/tests/library/locator-generator.spec.ts @@ -227,12 +227,6 @@ it('reverse engineer getByRole', async ({ page }) => { java: `getByRole(AriaRole.BUTTON, new Page.GetByRoleOptions().setChecked(true).setLevel(3).setPressed(false))`, csharp: `GetByRole(AriaRole.Button, new() { Checked = true, Level = 3, Pressed = false })`, }); - expect.soft(generate(page.getByRole('cell', { busy: false }))).toEqual({ - javascript: `getByRole('cell', { busy: false })`, - python: `get_by_role("cell", busy=False)`, - java: `getByRole(AriaRole.CELL, new Page.GetByRoleOptions().setBusy(false))`, - csharp: `GetByRole(AriaRole.Cell, new() { Busy = false })`, - }); expect.soft(generate(page.getByRole('alert', { name: 'Upload', description: 'doc.pdf' }))).toEqual({ javascript: `getByRole('alert', { name: 'Upload', description: 'doc.pdf' })`, python: `get_by_role("alert", name="Upload", description="doc.pdf")`, diff --git a/tests/library/screenshot.spec.ts b/tests/library/screenshot.spec.ts index 3403ba6ff3614..c2784f32a0ef5 100644 --- a/tests/library/screenshot.spec.ts +++ b/tests/library/screenshot.spec.ts @@ -210,8 +210,9 @@ browserTest.describe('page screenshot', () => { expect(pixel(0, 999).b).toBeGreaterThan(128); }); - browserTest('should not hang when event loop is blocked', { annotation: { type: 'issue', description: 'https://github.com/microsoft/playwright/issues/36702' } }, async ({ page, trace }) => { + browserTest('should not hang when event loop is blocked', { annotation: { type: 'issue', description: 'https://github.com/microsoft/playwright/issues/36702' } }, async ({ page, trace, mode }) => { browserTest.skip(trace === 'on', 'taking a snapshot hangs when the page is blocked'); + browserTest.skip(mode === 'driver', 'test hooks are not available in driver mode'); browserTest.setTimeout(5000); const __testHookBeforeScreenshot = async () => { page.evaluate(() => { diff --git a/tests/page/selectors-role.spec.ts b/tests/page/selectors-role.spec.ts index 10b4b353a0fe1..f7db3bcf105c1 100644 --- a/tests/page/selectors-role.spec.ts +++ b/tests/page/selectors-role.spec.ts @@ -208,49 +208,6 @@ test('should support expanded', async ({ page }) => { ]); }); -test('should support busy', async ({ page }) => { - await page.setContent(` -
Hi
-
Hello
-
Bye
- - - `); - - expect(await page.locator(`role=cell`).evaluateAll(els => els.map(e => e.outerHTML))).toEqual([ - `
Hi
`, - `
Hello
`, - `
Bye
`, - ]); - - expect(await page.locator(`role=cell[busy]`).evaluateAll(els => els.map(e => e.outerHTML))).toEqual([ - `
Hello
`, - ]); - expect(await page.locator(`role=cell[busy=true]`).evaluateAll(els => els.map(e => e.outerHTML))).toEqual([ - `
Hello
`, - ]); - expect(await page.getByRole('cell', { busy: true }).evaluateAll(els => els.map(e => e.outerHTML))).toEqual([ - `
Hello
`, - ]); - - expect(await page.locator(`role=cell[busy=false]`).evaluateAll(els => els.map(e => e.outerHTML))).toEqual([ - `
Hi
`, - `
Bye
`, - ]); - expect(await page.getByRole('cell', { busy: false }).evaluateAll(els => els.map(e => e.outerHTML))).toEqual([ - `
Hi
`, - `
Bye
`, - ]); - - // aria-busy is a global ARIA state and should work for any role. - expect(await page.getByRole('button', { busy: true }).evaluateAll(els => els.map(e => e.outerHTML))).toEqual([ - ``, - ]); - expect(await page.getByRole('button', { busy: false }).evaluateAll(els => els.map(e => e.outerHTML))).toEqual([ - ``, - ]); -}); - test('should support disabled', async ({ page }) => { await page.setContent(` @@ -554,7 +511,7 @@ test('errors', async ({ page }) => { expect(e0.message).toContain(`Role must not be empty`); const e1 = await page.$('role=foo[sElected]').catch(e => e); - expect(e1.message).toContain(`Unknown attribute "sElected", must be one of "busy", "checked", "description", "disabled", "expanded", "include-hidden", "level", "name", "pressed", "selected"`); + expect(e1.message).toContain(`Unknown attribute "sElected", must be one of "checked", "description", "disabled", "expanded", "include-hidden", "level", "name", "pressed", "selected"`); const e2 = await page.$('role=foo[bar . qux=true]').catch(e => e); expect(e2.message).toContain(`Unknown attribute "bar.qux"`); diff --git a/tests/playwright-test/reporter-html.spec.ts b/tests/playwright-test/reporter-html.spec.ts index 5f84bdaabe10f..707c4c04b25e1 100644 --- a/tests/playwright-test/reporter-html.spec.ts +++ b/tests/playwright-test/reporter-html.spec.ts @@ -3508,6 +3508,9 @@ for (const useIntermediateMergeReport of [true, false] as const) { test('should support merge files option', async ({ runInlineTest, showReport, page }) => { await runInlineTest({ + 'playwright.config.ts': ` + export default { reporter: [['html', { mergeFiles: true }], ['line']] }; + `, 'a.test.js': ` import { test, expect } from '@playwright/test'; test.describe('describe', () => { @@ -3521,13 +3524,10 @@ test('should support merge files option', async ({ runInlineTest, showReport, pa test('test 3', async ({}) => {}); }); `, - }, { reporter: 'dot,html' }, { PLAYWRIGHT_HTML_OPEN: 'never' }); + }, {}, { PLAYWRIGHT_HTML_OPEN: 'never' }); await showReport(); - await page.getByRole('button', { name: 'Settings' }).click(); - await page.getByRole('checkbox', { name: 'Merge files' }).click(); - await expect(page).toMatchAriaSnapshot(` - button "" [expanded] - region: diff --git a/tests/playwright-test/reporter-preprocess-suite.spec.ts b/tests/playwright-test/reporter-preprocess.spec.ts similarity index 78% rename from tests/playwright-test/reporter-preprocess-suite.spec.ts rename to tests/playwright-test/reporter-preprocess.spec.ts index d9139a34c518d..12930c32f57d9 100644 --- a/tests/playwright-test/reporter-preprocess-suite.spec.ts +++ b/tests/playwright-test/reporter-preprocess.spec.ts @@ -16,15 +16,15 @@ import { test, expect } from './playwright-test-fixtures'; -test('preprocessSuite sees the filtered corpus, can skip tests, and records the caller location', async ({ runInlineTest }) => { - // preprocessSuite runs between project setup and onBegin and sees the .only-narrowed corpus. +test('preprocess sees the filtered corpus, can skip tests, and records the caller location', async ({ runInlineTest }) => { + // preprocess runs between project setup and onBegin and sees the .only-narrowed corpus. const only = await runInlineTest({ 'reporter.ts': ` class Reporter { - async preprocessSuite(config, suite) { + async preprocess({ config, suite, testRun }) { console.log('%% plan: ' + suite.allTests().map(t => t.title).join(',')); for (const t of suite.allTests()) - if (t.title.includes('skip-me')) t.skip('planned skip'); + if (t.title.includes('skip-me')) testRun.skip(t, 'planned skip'); } onBegin(config, suite) { console.log('%% onBegin: ' + suite.allTests().map(t => t.title).join(',')); @@ -51,15 +51,15 @@ test('preprocessSuite sees the filtered corpus, can skip tests, and records the 'plan: run-me,skip-me', 'onBegin: run-me,skip-me', 'end run-me status=passed expected=passed ann= loc=none', - // The skip annotation location points at the reporter's `t.skip(...)` call (line 6 of reporter.ts). + // The skip annotation location points at the reporter's `testRun.skip(...)` call (line 6 of reporter.ts). 'end skip-me status=skipped expected=skipped ann=skip:planned skip loc=reporter.ts:6', ]); - // preprocessSuite respects --grep. + // preprocess respects --grep. const grep = await runInlineTest({ 'reporter.ts': ` class Reporter { - async preprocessSuite(config, suite) { + async preprocess({ config, suite }) { console.log('%% plan: ' + suite.allTests().map(t => t.title).join(',')); } } @@ -75,11 +75,11 @@ test('preprocessSuite sees the filtered corpus, can skip tests, and records the expect(grep.exitCode).toBe(0); expect(grep.outputLines).toEqual(['plan: foo-one']); - // preprocessSuite respects --project. + // preprocess respects --project. const project = await runInlineTest({ 'reporter.ts': ` class Reporter { - async preprocessSuite(config, suite) { + async preprocess({ config, suite }) { console.log('%% plan projects: ' + suite.suites.map(s => s.title).join(',')); } } @@ -104,11 +104,11 @@ test('TestCase.exclude and Suite.exclude remove entries from the run and report' const result = await runInlineTest({ 'reporter.ts': ` class Reporter { - async preprocessSuite(config, suite) { + async preprocess({ config, suite, testRun }) { for (const t of suite.allTests()) - if (t.title === 'excluded-test') t.exclude(); + if (t.title === 'excluded-test') testRun.exclude(t); const visit = (s) => { - if (s.title === 'excluded-suite') s.exclude(); + if (s.title === 'excluded-suite') testRun.exclude(s); else for (const child of s.suites || []) visit(child); }; visit(suite); @@ -144,9 +144,9 @@ test('Suite.skip cascades to all descendants', async ({ runInlineTest }) => { const result = await runInlineTest({ 'reporter.ts': ` class Reporter { - async preprocessSuite(config, suite) { + async preprocess({ config, suite, testRun }) { const visit = (s) => { - if (s.title === 'doomed') s.skip('whole group'); + if (s.title === 'doomed') testRun.skip(s, 'whole group'); for (const child of s.suites || []) visit(child); }; visit(suite); @@ -176,14 +176,15 @@ test('Suite.skip cascades to all descendants', async ({ runInlineTest }) => { ]); }); -test('disposition methods throw when called outside preprocessSuite, and the root suite cannot be excluded', async ({ runInlineTest }) => { +test('TestRun methods throw outside preprocess, and the root suite cannot be excluded', async ({ runInlineTest }) => { const result = await runInlineTest({ 'reporter.ts': ` class Reporter { - async preprocessSuite(config, suite) { - // Excluding the root suite is banned even during preprocessSuite. + async preprocess({ config, suite, testRun }) { + this.testRun = testRun; + // Excluding the root suite is banned even during preprocess. try { - suite.exclude(); + testRun.exclude(suite); console.log('%% root-exclude: no-throw'); } catch (e) { console.log('%% root-exclude: ' + e.message); @@ -195,7 +196,7 @@ test('disposition methods throw when called outside preprocessSuite, and the roo for (const [label, obj] of [['TestCase', testCase], ['Suite', fileSuite]]) { for (const method of ['skip', 'fixme', 'fail', 'exclude']) { try { - obj[method](); + this.testRun[method](obj); console.log('%% ' + label + '.' + method + ': no-throw'); } catch (e) { console.log('%% ' + label + '.' + method + ': ' + e.message); @@ -215,23 +216,23 @@ test('disposition methods throw when called outside preprocessSuite, and the roo expect(result.exitCode).toBe(0); expect(result.outputLines).toEqual([ - 'root-exclude: Suite.exclude() cannot be called on the root suite.', - 'TestCase.skip: TestCase.skip() can only be called from Reporter.preprocessSuite().', - 'TestCase.fixme: TestCase.fixme() can only be called from Reporter.preprocessSuite().', - 'TestCase.fail: TestCase.fail() can only be called from Reporter.preprocessSuite().', - 'TestCase.exclude: TestCase.exclude() can only be called from Reporter.preprocessSuite().', - 'Suite.skip: Suite.skip() can only be called from Reporter.preprocessSuite().', - 'Suite.fixme: Suite.fixme() can only be called from Reporter.preprocessSuite().', - 'Suite.fail: Suite.fail() can only be called from Reporter.preprocessSuite().', - 'Suite.exclude: Suite.exclude() can only be called from Reporter.preprocessSuite().', + 'root-exclude: TestRun.exclude() cannot be called on the root suite.', + 'TestCase.skip: TestRun.skip() can only be called from Reporter.preprocess().', + 'TestCase.fixme: TestRun.fixme() can only be called from Reporter.preprocess().', + 'TestCase.fail: TestRun.fail() can only be called from Reporter.preprocess().', + 'TestCase.exclude: TestRun.exclude() can only be called from Reporter.preprocess().', + 'Suite.skip: TestRun.skip() can only be called from Reporter.preprocess().', + 'Suite.fixme: TestRun.fixme() can only be called from Reporter.preprocess().', + 'Suite.fail: TestRun.fail() can only be called from Reporter.preprocess().', + 'Suite.exclude: TestRun.exclude() can only be called from Reporter.preprocess().', ]); }); -test('preprocessSuite throwing aborts the run before onBegin', async ({ runInlineTest }) => { +test('preprocess throwing aborts the run before onBegin', async ({ runInlineTest }) => { const result = await runInlineTest({ 'reporter.ts': ` class Reporter { - async preprocessSuite(config, suite) { + async preprocess({ config, suite }) { throw new Error('plan-aborted'); } onBegin(config, suite) { @@ -256,15 +257,15 @@ test('preprocessSuite throwing aborts the run before onBegin', async ({ runInlin expect(result.outputLines).not.toContain('onBegin: 1'); }); -test('multiple reporters: preprocessSuite called in order, annotations accumulate, exclude prunes for next reporter', async ({ runInlineTest }) => { +test('multiple reporters: preprocess called in order, annotations accumulate, exclude prunes for next reporter', async ({ runInlineTest }) => { const result = await runInlineTest({ 'first.ts': ` class R { - async preprocessSuite(config, suite) { + async preprocess({ config, suite, testRun }) { console.log('%% first plan sees: ' + suite.allTests().map(t => t.title).join(',')); for (const t of suite.allTests()) { - if (t.title === 'gone') t.exclude(); - else t.fail('first reason'); + if (t.title === 'gone') testRun.exclude(t); + else testRun.fail(t, 'first reason'); } } onTestEnd(test, result) { @@ -275,9 +276,9 @@ test('multiple reporters: preprocessSuite called in order, annotations accumulat `, 'second.ts': ` class R { - async preprocessSuite(config, suite) { + async preprocess({ config, suite, testRun }) { console.log('%% second plan sees: ' + suite.allTests().map(t => t.title).join(',')); - suite.allTests()[0].skip('second reason'); + testRun.skip(suite.allTests()[0], 'second reason'); } } module.exports = R; @@ -303,15 +304,15 @@ test('multiple reporters: a later reporter observes an earlier reporter Suite.sk const result = await runInlineTest({ 'first.ts': ` class R { - async preprocessSuite(config, suite) { - suite.allTests()[0].parent.skip('first reason'); + async preprocess({ config, suite, testRun }) { + testRun.skip(suite.allTests()[0].parent, 'first reason'); } } module.exports = R; `, 'second.ts': ` class R { - async preprocessSuite(config, suite) { + async preprocess({ config, suite }) { const skipped = suite.allTests().filter(t => t.expectedStatus === 'skipped').map(t => t.title); console.log('%% second sees skipped: ' + skipped.join(',')); } @@ -333,18 +334,18 @@ test('multiple reporters: a later reporter observes an earlier reporter Suite.sk expect(result.outputLines).toContain('second sees skipped: one,two'); }); -test('implementsSharding disables the built-in shard filter; preprocessSuite sees the full corpus', async ({ runInlineTest }) => { +test('skipSharding disables the built-in shard filter; preprocess sees the full corpus', async ({ runInlineTest }) => { const result = await runInlineTest({ 'reporter.ts': ` class R { - async preprocessSuite(config, suite) { - // preprocessSuite observes the full, un-sharded corpus regardless of --shard. + async preprocess({ config, suite, testRun }) { + // preprocess observes the full, un-sharded corpus regardless of --shard. console.log('%% plan: ' + suite.allTests().map(t => t.title).join(',')); + testRun.skipSharding(); let i = 0; for (const t of suite.allTests()) { - if (i++ % 2 === 1) t.exclude(); + if (i++ % 2 === 1) testRun.exclude(t); } - return { implementsSharding: true }; } onBegin(config, suite) { console.log('%% begin: ' + suite.allTests().map(t => t.title).join(',')); @@ -361,24 +362,24 @@ test('implementsSharding disables the built-in shard filter; preprocessSuite see }, { reporter: '', workers: 1 }); expect(result.exitCode).toBe(0); - // preprocessSuite sees all four tests even though --shard=1/2 was configured. + // preprocess sees all four tests even though --shard=1/2 was configured. expect(result.outputLines).toContain('plan: t0,t1,t2,t3'); // The reporter's own exclusions define the shard; the built-in shard filter did NOT run // (it would have produced a different split), so t0,t2 remain. expect(result.outputLines).toContain('begin: t0,t2'); }); -test('multiple reporters declaring implementsSharding throws', async ({ runInlineTest }) => { +test('multiple reporters declaring custom sharding throws', async ({ runInlineTest }) => { const result = await runInlineTest({ 'reporter-a.ts': ` class A { - preprocessSuite() { return { implementsSharding: true }; } + preprocess({ testRun }) { testRun.skipSharding(); } onError(err) { console.log('%% error: ' + err.message); } } module.exports = A; `, 'reporter-b.ts': ` - class B { preprocessSuite() { return { implementsSharding: true }; } } + class B { preprocess({ testRun }) { testRun.skipSharding(); } } module.exports = B; `, 'playwright.config.ts': `module.exports = { reporter: [['./reporter-a.ts'], ['./reporter-b.ts']] };`, @@ -389,21 +390,22 @@ test('multiple reporters declaring implementsSharding throws', async ({ runInlin }, { reporter: '', workers: 1 }); expect(result.exitCode).not.toBe(0); - expect(result.outputLines.join('\n')).toContain(`Multiple reporters declare 'implementsSharding'`); + expect(result.outputLines.join('\n')).toContain(`Multiple reporters called 'skipSharding'`); }); test('plan.suite exposes setup/teardown dependency projects but they are read-only', async ({ runInlineTest }) => { const result = await runInlineTest({ 'reporter.ts': ` class Reporter { - async preprocessSuite(config, suite) { + async preprocess({ config, suite, testRun }) { + this.testRun = testRun; console.log('%% plan projects: ' + suite.suites.map(s => s.title).join(',')); console.log('%% plan tests: ' + suite.allTests().map(t => t.title).join(',')); this.preprocessedTests = new Set(suite.allTests()); const setupTest = suite.allTests().find(t => t.title === 'setup-test'); for (const method of ['skip', 'fixme', 'fail', 'exclude']) { try { - setupTest[method](); + testRun[method](setupTest); console.log('%% dep-' + method + ': no-throw'); } catch (e) { console.log('%% dep-' + method + ': ' + e.message); @@ -411,7 +413,7 @@ test('plan.suite exposes setup/teardown dependency projects but they are read-on } const setupProject = suite.suites.find(s => s.title === 'setup'); try { - setupProject.exclude(); + testRun.exclude(setupProject); console.log('%% dep-suite-exclude: no-throw'); } catch (e) { console.log('%% dep-suite-exclude: ' + e.message); @@ -421,7 +423,7 @@ test('plan.suite exposes setup/teardown dependency projects but they are read-on console.log('%% same test objects: ' + suite.allTests().every(test => this.preprocessedTests.has(test))); const setupTest = suite.allTests().find(t => t.title === 'setup-test'); try { - setupTest.skip(); + this.testRun.skip(setupTest); console.log('%% dep-after-preprocess: no-throw'); } catch (e) { console.log('%% dep-after-preprocess: ' + e.message); @@ -461,13 +463,13 @@ test('plan.suite exposes setup/teardown dependency projects but they are read-on expect(result.outputLines).toEqual([ 'plan projects: teardown,setup,main', 'plan tests: teardown-test,setup-test,main-test', - 'dep-skip: TestCase.skip() cannot be called on a setup or teardown project test; these always run in full.', - 'dep-fixme: TestCase.fixme() cannot be called on a setup or teardown project test; these always run in full.', - 'dep-fail: TestCase.fail() cannot be called on a setup or teardown project test; these always run in full.', - 'dep-exclude: TestCase.exclude() cannot be called on a setup or teardown project test; these always run in full.', - 'dep-suite-exclude: Suite.exclude() cannot be called on a setup or teardown project; these always run in full.', + 'dep-skip: TestRun.skip() cannot be called on a setup or teardown project test; these always run in full.', + 'dep-fixme: TestRun.fixme() cannot be called on a setup or teardown project test; these always run in full.', + 'dep-fail: TestRun.fail() cannot be called on a setup or teardown project test; these always run in full.', + 'dep-exclude: TestRun.exclude() cannot be called on a setup or teardown project test; these always run in full.', + 'dep-suite-exclude: TestRun.exclude() cannot be called on a setup or teardown project; these always run in full.', 'same test objects: true', - 'dep-after-preprocess: TestCase.skip() can only be called from Reporter.preprocessSuite().', + 'dep-after-preprocess: TestRun.skip() can only be called from Reporter.preprocess().', 'ran setup/setup-test', 'ran main/main-test', 'ran teardown/teardown-test', @@ -478,9 +480,9 @@ test('plan.suite temporarily exposes dependencies without changing final project const result = await runInlineTest({ 'reporter.ts': ` class Reporter { - async preprocessSuite(config, suite) { + async preprocess({ config, suite, testRun }) { console.log('%% plan projects: ' + suite.suites.map(suite => suite.title).join(',')); - suite.suites.find(suite => suite.title === 'main').exclude(); + testRun.exclude(suite.suites.find(suite => suite.title === 'main')); } onTestEnd(test, result) { console.log('%% ran ' + test.parent.project().name + '/' + test.title); diff --git a/tests/playwright-test/retry.spec.ts b/tests/playwright-test/retry.spec.ts index 1800eaded07d8..a251d228d3e4f 100644 --- a/tests/playwright-test/retry.spec.ts +++ b/tests/playwright-test/retry.spec.ts @@ -263,10 +263,10 @@ test('failed and skipped on retry should be marked as flaky', async ({ runInline expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'Skipped on first retry', location: expect.anything() }]); }); -test('should defer retries to the end of the run', async ({ runInlineTest }) => { +test('should run isolated retries at the end of the run', async ({ runInlineTest }) => { const result = await runInlineTest({ 'playwright.config.js': ` - module.exports = { retries: 3, retryStrategy: 'deferred' }; + module.exports = { retries: 3, retryStrategy: 'isolated' }; `, 'a.test.js': ` import { test, expect } from '@playwright/test'; @@ -298,3 +298,72 @@ test('should defer retries to the end of the run', async ({ runInlineTest }) => 'a-3', ]); }); + +test('should run isolated retries one by one in a single worker', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.js': ` + module.exports = { retries: 1, retryStrategy: 'isolated', workers: 2 }; + `, + 'a.test.js': ` + import { test, expect } from '@playwright/test'; + test('a', ({}, testInfo) => { + console.log('\\n%%a-' + testInfo.retry + '/' + testInfo.workerIndex); + expect(testInfo.retry).toBe(1); + }); + `, + 'b.test.js': ` + import { test, expect } from '@playwright/test'; + test('b', ({}, testInfo) => { + console.log('\\n%%b-' + testInfo.retry + '/' + testInfo.workerIndex); + expect(testInfo.retry).toBe(1); + }); + `, + }); + expect(result.exitCode).toBe(0); + expect(result.flaky).toBe(2); + expect(result.results.length).toBe(4); + const lines = result.outputLines; + // First attempts run in parallel workers, before any retry. + expect(lines.slice(0, 2).map(line => line.split('/')[0]).sort()).toEqual(['a-0', 'b-0']); + expect(lines.slice(2).map(line => line.split('/')[0]).sort()).toEqual(['a-1', 'b-1']); + // Both retries run in the same worker, one by one. + const retryWorkers = lines.slice(2).map(line => line.split('/')[1]); + expect(retryWorkers[0]).toBe(retryWorkers[1]); +}); + +test('should not defer remaining tests with isolated retries', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.js': ` + module.exports = { retries: 1, retryStrategy: 'isolated' }; + `, + 'a.test.js': ` + import { test, expect } from '@playwright/test'; + test('a1', ({}, testInfo) => { + console.log('\\n%%a1-' + testInfo.retry); + expect(testInfo.retry).toBe(1); + }); + test('a2', ({}, testInfo) => { + console.log('\\n%%a2-' + testInfo.retry); + }); + `, + 'b.test.js': ` + import { test, expect } from '@playwright/test'; + test('b1', ({}, testInfo) => { + console.log('\\n%%b1-' + testInfo.retry); + expect(testInfo.retry).toBe(1); + }); + `, + }, { workers: 1 }); + expect(result.exitCode).toBe(0); + expect(result.flaky).toBe(2); + expect(result.passed).toBe(1); + // After "a1" fails, "a2" keeps its priority in the queue and runs + // before "b1", while the retry of "a1" is deferred to the end. + expect(result.outputLines).toEqual([ + 'a1-0', + 'a2-0', + 'b1-0', + 'a1-1', + 'b1-1', + ]); +}); diff --git a/tests/playwright-test/ui-mode-test-filters.spec.ts b/tests/playwright-test/ui-mode-test-filters.spec.ts index fefe28f03e104..ef72b00b5759a 100644 --- a/tests/playwright-test/ui-mode-test-filters.spec.ts +++ b/tests/playwright-test/ui-mode-test-filters.spec.ts @@ -175,6 +175,32 @@ test('should filter by project', async ({ runUITest }) => { await expect(page.getByText('Projects: foo bar')).toBeVisible(); }); +test('should list projects in config order', async ({ runUITest }) => { + test.info().annotations.push({ type: 'issue', description: 'https://github.com/microsoft/playwright/issues/41779' }); + const { page } = await runUITest({ + ...basicTestTree, + 'playwright.config.ts': ` + import { defineConfig } from '@playwright/test'; + export default defineConfig({ + projects: [ + { name: 'dev-setup', testMatch: /a.test.ts/ }, + { name: 'dev', dependencies: ['dev-setup'] }, + { name: 'staging-setup', testMatch: /a.test.ts/ }, + { name: 'staging', dependencies: ['staging-setup'] }, + ], + }); + ` + }); + + await page.getByText('Status:').click(); + await expect(page.getByTestId('project-filters').locator('.filter-entry')).toHaveText([ + 'dev-setup', + 'dev', + 'staging-setup', + 'staging', + ]); +}); + test('should not hide filtered while running', async ({ runUITest, createLatch }) => { const latch = createLatch(); const { page } = await runUITest({ diff --git a/tests/playwright-test/ui-mode-test-setup.spec.ts b/tests/playwright-test/ui-mode-test-setup.spec.ts index 5738a3ec22af5..8b08d37985bc0 100644 --- a/tests/playwright-test/ui-mode-test-setup.spec.ts +++ b/tests/playwright-test/ui-mode-test-setup.spec.ts @@ -135,12 +135,12 @@ test('should run setup and teardown projects (1)', async ({ runUITest }) => { await expect(page.getByTestId('project-filters')).toMatchAriaSnapshot(` - list: - - listitem: - - checkbox "teardown" - listitem: - checkbox "setup" - listitem: - checkbox "test" + - listitem: + - checkbox "teardown" `); await page.getByTitle('Run all').click(); @@ -186,12 +186,12 @@ test('should run setup and teardown projects (2)', async ({ runUITest }) => { await expect(page.getByTestId('project-filters')).toMatchAriaSnapshot(` - list: - - listitem: - - checkbox "teardown" [checked] - listitem: - checkbox "setup" - listitem: - checkbox "test" [checked] + - listitem: + - checkbox "teardown" [checked] `); await page.getByTitle('Run all').click(); @@ -232,12 +232,12 @@ test('should run setup and teardown projects (3)', async ({ runUITest }) => { await expect(page.getByTestId('project-filters')).toMatchAriaSnapshot(` - list: - - listitem: - - checkbox "teardown" - listitem: - checkbox "setup" - listitem: - checkbox "test" [checked] + - listitem: + - checkbox "teardown" `); await page.getByTitle('Run all').click(); @@ -273,12 +273,12 @@ test('should run part of the setup only', async ({ runUITest }) => { await expect(page.getByTestId('project-filters')).toMatchAriaSnapshot(` - list: - - listitem: - - checkbox "teardown" [checked] - listitem: - checkbox "setup" [checked] - listitem: - checkbox "test" [checked] + - listitem: + - checkbox "teardown" [checked] `); await page.getByText('setup.ts').hover(); diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts index 7087c2dea2299..f06954a448433 100644 --- a/utils/generate_types/overrides-test.d.ts +++ b/utils/generate_types/overrides-test.d.ts @@ -31,6 +31,7 @@ export type HtmlReporterOptions = { noSnippets?: boolean; noCopyPrompt?: boolean; doNotInlineAssets?: boolean; + mergeFiles?: boolean; }; export type ReporterDescription = Readonly< diff --git a/utils/generate_types/overrides-testReporter.d.ts b/utils/generate_types/overrides-testReporter.d.ts index 26633625a568f..6ca13fdcf2157 100644 --- a/utils/generate_types/overrides-testReporter.d.ts +++ b/utils/generate_types/overrides-testReporter.d.ts @@ -42,7 +42,6 @@ export interface FullResult { } export interface Reporter { - preprocessSuite?(config: FullConfig, suite: Suite): Promise<{ implementsSharding?: boolean } | undefined | void> | { implementsSharding?: boolean } | void; onEnd?(result: FullResult): Promise<{ status?: FullResult['status'] } | undefined | void> | void; }