diff --git a/.claude/skills/playwright-triage/SKILL.md b/.claude/skills/playwright-triage/SKILL.md index 5b1ef32d36830..bed461bf3ccf8 100644 --- a/.claude/skills/playwright-triage/SKILL.md +++ b/.claude/skills/playwright-triage/SKILL.md @@ -58,7 +58,7 @@ result to report, not a non-finding. match. (A version ending in `-next`, e.g. `1.62.0-next`, is **not** an npm version — it means tip-of-tree, which is the `@next` build you already tried.) -To step through a test interactively, use the [playwright-cli](../playwright-cli/SKILL.md) skill. +To step through a test interactively, use the [playwright-cli](../../../packages/playwright-core/src/tools/skills/playwright-cli/SKILL.md) skill. Reports sometimes target another part of the Playwright project — `@playwright/mcp` (its source is in this repo), `playwright-vscode`, `playwright-python`, `playwright-java`, `playwright-dotnet`. diff --git a/docs/src/browsers.md b/docs/src/browsers.md index 47d88ef7bc391..41d1c81c58ae1 100644 --- a/docs/src/browsers.md +++ b/docs/src/browsers.md @@ -157,7 +157,7 @@ npx playwright --version ## Configure Browsers -Playwright can run tests on Chromium, WebKit and Firefox browsers as well as branded browsers such as Google Chrome and Microsoft Edge. It can also run on emulated tablet and mobile devices. See the [registry of device parameters](https://github.com/microsoft/playwright/blob/main/packages/playwright-core/src/server/deviceDescriptorsSource.json) for a complete list of selected desktop, tablet and mobile devices. +Playwright can run tests on Chromium, WebKit and Firefox browsers as well as branded browsers such as Google Chrome and Microsoft Edge. It can also run on emulated tablet and mobile devices. See the [registry of device parameters](https://github.com/microsoft/playwright/blob/main/packages/isomorphic/deviceDescriptorsSource.json) for a complete list of selected desktop, tablet and mobile devices. ### Run tests on different browsers * langs: js diff --git a/docs/src/emulation.md b/docs/src/emulation.md index bc9da7fe3fcbf..d05d07e6a255c 100644 --- a/docs/src/emulation.md +++ b/docs/src/emulation.md @@ -10,7 +10,7 @@ With Playwright you can test your app on any browser as well as emulate a real d ## Devices * langs: js, csharp, python -Playwright comes with a [registry of device parameters](https://github.com/microsoft/playwright/blob/main/packages/playwright-core/src/server/deviceDescriptorsSource.json) using [`property: Playwright.devices`] for selected desktop, tablet and mobile devices. It can be used to simulate browser behavior for a specific device such as user agent, screen size, viewport and if it has touch enabled. All tests will run with the specified device parameters. +Playwright comes with a [registry of device parameters](https://github.com/microsoft/playwright/blob/main/packages/isomorphic/deviceDescriptorsSource.json) using [`property: Playwright.devices`] for selected desktop, tablet and mobile devices. It can be used to simulate browser behavior for a specific device such as user agent, screen size, viewport and if it has touch enabled. All tests will run with the specified device parameters. ```js tab=js-test title="playwright.config.ts" import { defineConfig, devices } from '@playwright/test'; // import devices diff --git a/docs/src/getting-started-cli.md b/docs/src/getting-started-cli.md index 96a68fdb0c21f..e57793372497d 100644 --- a/docs/src/getting-started-cli.md +++ b/docs/src/getting-started-cli.md @@ -101,7 +101,7 @@ playwright-cli check # check a checkbox or radio button playwright-cli uncheck # uncheck a checkbox playwright-cli hover # hover over element playwright-cli drag # drag and drop between elements -playwright-cli upload # upload files +playwright-cli upload # upload one or multiple files playwright-cli close # close the page ``` diff --git a/docs/src/test-projects-js.md b/docs/src/test-projects-js.md index 6c16c18857e50..33cb7d1fce13f 100644 --- a/docs/src/test-projects-js.md +++ b/docs/src/test-projects-js.md @@ -11,7 +11,7 @@ By setting up projects you can also run a group of tests with different timeouts ## Configure projects for multiple browsers -By using **projects** you can run your tests in multiple browsers such as chromium, webkit and firefox as well as branded browsers such as Google Chrome and Microsoft Edge. Playwright can also run on emulated tablet and mobile devices. See the [registry of device parameters](https://github.com/microsoft/playwright/blob/main/packages/playwright-core/src/server/deviceDescriptorsSource.json) for a complete list of selected desktop, tablet and mobile devices. +By using **projects** you can run your tests in multiple browsers such as chromium, webkit and firefox as well as branded browsers such as Google Chrome and Microsoft Edge. Playwright can also run on emulated tablet and mobile devices. See the [registry of device parameters](https://github.com/microsoft/playwright/blob/main/packages/isomorphic/deviceDescriptorsSource.json) for a complete list of selected desktop, tablet and mobile devices. ```js import { defineConfig, devices } from '@playwright/test'; diff --git a/packages/isomorphic/trace/entries.ts b/packages/isomorphic/trace/entries.ts index 5fb51f63f4ca4..cd902c9f118c2 100644 --- a/packages/isomorphic/trace/entries.ts +++ b/packages/isomorphic/trace/entries.ts @@ -42,6 +42,7 @@ export type ContextEntry = { hasSource: boolean; contextId: string; testTimeout?: number; + annotations?: trace.TraceEventAnnotation[]; }; export type PageEntry = { diff --git a/packages/isomorphic/trace/traceModel.ts b/packages/isomorphic/trace/traceModel.ts index 70c236cd710c7..eaa75073bfc7c 100644 --- a/packages/isomorphic/trace/traceModel.ts +++ b/packages/isomorphic/trace/traceModel.ts @@ -89,6 +89,7 @@ export class TraceModel { readonly actionCounters: Map; readonly traceUri: string; readonly testTimeout?: number; + readonly annotations?: trace.TraceEventAnnotation[]; readonly pagerefToTitle = new Map(); readonly contextToTitle = new Map(); @@ -106,6 +107,7 @@ export class TraceModel { this.title = libraryContext?.title || ''; this.options = libraryContext?.options || {}; this.testTimeout = contexts.find(c => c.origin === 'testRunner')?.testTimeout; + this.annotations = contexts.find(c => c.origin === 'testRunner')?.annotations; // Next call updates all timestamps for all events in library contexts, so it must be done first. this.actions = mergeActionsAndUpdateTiming(contexts); this.pages = ([] as PageEntry[]).concat(...contexts.map(c => c.pages)); diff --git a/packages/isomorphic/trace/traceModernizer.ts b/packages/isomorphic/trace/traceModernizer.ts index 20e703145f553..b18662b16e24b 100644 --- a/packages/isomorphic/trace/traceModernizer.ts +++ b/packages/isomorphic/trace/traceModernizer.ts @@ -99,6 +99,7 @@ export class TraceModernizer { contextEntry.testIdAttributeName = event.testIdAttributeName; contextEntry.contextId = event.contextId ?? ''; contextEntry.testTimeout = event.testTimeout; + contextEntry.annotations = event.annotations; break; } case 'screencast-frame': { diff --git a/packages/playwright-core/browsers.json b/packages/playwright-core/browsers.json index 4e49c41d1fdb0..9e0d42534398f 100644 --- a/packages/playwright-core/browsers.json +++ b/packages/playwright-core/browsers.json @@ -3,14 +3,14 @@ "browsers": [ { "name": "chromium", - "revision": "1235", + "revision": "1236", "installByDefault": true, "browserVersion": "151.0.7922.47", "title": "Chrome for Testing" }, { "name": "chromium-headless-shell", - "revision": "1235", + "revision": "1236", "installByDefault": true, "browserVersion": "151.0.7922.47", "title": "Chrome Headless Shell" @@ -24,7 +24,7 @@ }, { "name": "webkit", - "revision": "2340", + "revision": "2341", "installByDefault": true, "revisionOverrides": { "mac14": "2251", diff --git a/packages/playwright-core/src/server/bidi/bidiBrowser.ts b/packages/playwright-core/src/server/bidi/bidiBrowser.ts index 95546d70fe53c..c4f2280baf469 100644 --- a/packages/playwright-core/src/server/bidi/bidiBrowser.ts +++ b/packages/playwright-core/src/server/bidi/bidiBrowser.ts @@ -331,8 +331,11 @@ export class BidiBrowserContext extends BrowserContext { async doGrantPermissions(origin: string, permissions: string[]) { if (origin === 'null') return; + const protocolPermissions = permissions.flatMap( + permission => permission === 'local-network-access' ? ['local-network', 'loopback-network'] : permission + ); const currentPermissions = this._originToPermissions.get(origin) || []; - const toGrant = permissions.filter(permission => !currentPermissions.includes(permission)); + const toGrant = protocolPermissions.filter(permission => !currentPermissions.includes(permission)); this._originToPermissions.set(origin, [...currentPermissions, ...toGrant]); if (origin === '*') { await Promise.all(this._bidiPages().flatMap(page => diff --git a/packages/playwright-core/src/server/webkit/protocol.d.ts b/packages/playwright-core/src/server/webkit/protocol.d.ts index 922ef8a9c62c4..f7ccf4086bc13 100644 --- a/packages/playwright-core/src/server/webkit/protocol.d.ts +++ b/packages/playwright-core/src/server/webkit/protocol.d.ts @@ -3571,6 +3571,7 @@ might return multiple quads for inline nodes. * Unique script identifier. */ export type ScriptId = string; + export type ScriptType = "program"|"module"|"webassembly"; /** * Call frame identifier. */ @@ -3588,7 +3589,7 @@ might return multiple quads for inline nodes. */ lineNumber: number; /** - * Column number in the script (0-based). + * Column number in the script (0-based) or bytecode offset for WebAssembly modules (0-based). */ columnNumber?: number; } @@ -3791,9 +3792,17 @@ might return multiple quads for inline nodes. */ endLine: number; /** - * Length of the last line of the script. + * Length of the last line of the script or the end bytecode offset for WebAssembly modules. */ endColumn: number; + /** + * Identifier of the execution context in which this script was parsed. + */ + executionContextId: Runtime.ExecutionContextId; + /** + * Type of script. + */ + scriptType: ScriptType; /** * Determines whether this script is a user extension script. */ @@ -3807,9 +3816,13 @@ might return multiple quads for inline nodes. */ sourceMapURL?: string; /** - * True if this script was parsed as a module. + * Human-readable name of the script. */ - module?: boolean; + displayName?: string; + /** + * Identifier of the network request associated with this script (if any). + */ + requestId?: Network.RequestId; } /** * Fired when virtual machine fails to parse the script. diff --git a/packages/playwright-core/src/tools/cli-client/program.ts b/packages/playwright-core/src/tools/cli-client/program.ts index 6cf1a6b37e043..6a087c48c1653 100644 --- a/packages/playwright-core/src/tools/cli-client/program.ts +++ b/packages/playwright-core/src/tools/cli-client/program.ts @@ -295,6 +295,8 @@ async function runInSession(entry: SessionFile, clientInfo: ClientInfo, args: Mi delete args[globalOption]; const session = new Session(entry); const result = await session.run(clientInfo, args, { raw, json: output.json }); + if (result.isError) + process.exitCode = 1; return result.text; } @@ -445,9 +447,9 @@ function validateFlags(args: MinimistArgs, command: { flags: Record command.args.length) + if (positional.length > command.args.length && !command.variadicArg) output.errorTooManyArguments(command.args.length, positional.length, command.help); } diff --git a/packages/playwright-core/src/tools/cli-client/session.ts b/packages/playwright-core/src/tools/cli-client/session.ts index bbcf54cf41d90..2db6a105e222b 100644 --- a/packages/playwright-core/src/tools/cli-client/session.ts +++ b/packages/playwright-core/src/tools/cli-client/session.ts @@ -42,7 +42,7 @@ export class Session { return compareSemver(clientInfo.version, this.config.version) >= 0; } - async run(clientInfo: ClientInfo, args: MinimistArgs, options?: { raw?: boolean, json?: boolean }): Promise<{ text: string }> { + async run(clientInfo: ClientInfo, args: MinimistArgs, options?: { raw?: boolean, json?: boolean }): Promise<{ text: string, isError?: boolean }> { if (!this.isCompatible(clientInfo)) throw new Error(`Client is v${clientInfo.version}, session '${this.name}' is v${this.config.version}. Run\n\n playwright-cli${this.name !== 'default' ? ` -s=${this.name}` : ''} open\n\nto restart the browser session.`); diff --git a/packages/playwright-core/src/tools/cli-daemon/command.ts b/packages/playwright-core/src/tools/cli-daemon/command.ts index 4c14bba89d17d..96ae8eed699ac 100644 --- a/packages/playwright-core/src/tools/cli-daemon/command.ts +++ b/packages/playwright-core/src/tools/cli-daemon/command.ts @@ -40,6 +40,19 @@ export function declareCommand isVariadicArg(option as zodType.ZodTypeAny)); + if (schema instanceof z.ZodOptional) + return isVariadicArg(schema.unwrap() as zodType.ZodTypeAny); + if (schema instanceof z.ZodPipe) + return isVariadicArg(schema.in as zodType.ZodTypeAny); + return false; +} + export function parseCommand(command: AnyCommandSchema, args: Record & { _: string[] }): { toolName: string, toolParams: any } { const optionsObject = { ...args } as Record; delete optionsObject['_']; @@ -49,11 +62,17 @@ export function parseCommand(command: AnyCommandSchema, args: Record argNames.length) + const variadic = argNames.length > 0 && isVariadicArg(argsSchema.shape[argNames[argNames.length - 1]]); + if (argv.length > argNames.length && !variadic) throw new Error(`error: too many arguments: expected ${argNames.length}, received ${argv.length}`); - const argsObject: Record = {}; - argNames.forEach((name, index) => argsObject[name] = argv[index]); - const parsedArgsObject: Record = zodParse(argsSchema, argsObject, 'argument'); + const argsObject: Record = {}; + argNames.forEach((name, index) => { + if (variadic && index === argNames.length - 1) + argsObject[name] = index < argv.length ? argv.slice(index) : undefined; + else + argsObject[name] = argv[index]; + }); + const parsedArgsObject: Record = zodParse(argsSchema, argsObject, 'argument'); const toolName = typeof command.toolName === 'function' ? command.toolName({ ...parsedArgsObject, ...options }) : command.toolName; const toolParams = command.toolParams({ ...parsedArgsObject, ...options }); @@ -72,6 +91,10 @@ function zodParse(schema: zodType.ZodAny, data: unknown, type: 'option' | 'argum switch (issue.code) { case 'invalid_type': return 'error: ' + label + ': ' + issue.message.replace(/Invalid input:/, '').trim(); + case 'invalid_union': { + const message = issue.errors[0]?.[0]?.message ?? issue.message; + return 'error: ' + label + ': ' + message.replace(/Invalid input:/, '').trim(); + } case 'unrecognized_keys': return 'error: unknown ' + label; default: diff --git a/packages/playwright-core/src/tools/cli-daemon/commands.ts b/packages/playwright-core/src/tools/cli-daemon/commands.ts index 41b95f9ee37f2..53cd5642852fd 100644 --- a/packages/playwright-core/src/tools/cli-daemon/commands.ts +++ b/packages/playwright-core/src/tools/cli-daemon/commands.ts @@ -343,10 +343,10 @@ const fileUpload = declareCommand({ description: 'Upload one or multiple files', category: 'core', args: z.object({ - file: z.string().describe('The absolute paths to the files to upload'), + files: stringArrayArg.describe('The absolute paths to the files to upload'), }), toolName: 'browser_file_upload', - toolParams: ({ file }) => ({ paths: [file] }), + toolParams: ({ files }) => ({ paths: files }), }); const check = declareCommand({ diff --git a/packages/playwright-core/src/tools/cli-daemon/helpGenerator.ts b/packages/playwright-core/src/tools/cli-daemon/helpGenerator.ts index 7590700a16382..00925fd0bd82e 100644 --- a/packages/playwright-core/src/tools/cli-daemon/helpGenerator.ts +++ b/packages/playwright-core/src/tools/cli-daemon/helpGenerator.ts @@ -16,25 +16,33 @@ import * as z from 'zod'; import { commands } from './commands'; +import { isVariadicArg } from './command'; import type zodType from 'zod'; import type { AnyCommandSchema, Category } from './command'; -type CommandArg = { name: string, description: string, optional: boolean }; +type CommandArg = { name: string, description: string, optional: boolean, variadic: boolean }; function commandArgs(command: AnyCommandSchema): CommandArg[] { const args: CommandArg[] = []; const shape = command.args ? (command.args as zodType.ZodObject).shape : {}; + const names = Object.keys(shape); for (const [name, schema] of Object.entries(shape)) { const zodSchema = schema as zodType.ZodTypeAny; const description = zodSchema.description ?? ''; - args.push({ name, description, optional: zodSchema.safeParse(undefined).success }); + const variadic = name === names[names.length - 1] && isVariadicArg(zodSchema); + args.push({ name, description, optional: zodSchema.safeParse(undefined).success, variadic }); } return args; } +function commandArgText(a: CommandArg) { + const name = a.variadic ? `${a.name}...` : a.name; + return a.optional ? `[${name}]` : `<${name}>`; +} + function commandArgsText(args: CommandArg[]) { - return args.map(a => a.optional ? `[${a.name}]` : `<${a.name}>`).join(' '); + return args.map(commandArgText).join(' '); } function generateCommandHelp(command: AnyCommandSchema) { @@ -48,7 +56,7 @@ function generateCommandHelp(command: AnyCommandSchema) { if (args.length) { lines.push('Arguments:'); - lines.push(...args.map(a => formatWithGap(` ${a.optional ? `[${a.name}]` : `<${a.name}>`}`, a.description.toLowerCase()))); + lines.push(...args.map(a => formatWithGap(` ${commandArgText(a)}`, a.description.toLowerCase()))); } if (command.options) { @@ -165,7 +173,7 @@ function isBooleanSchema(schema: zodType.ZodTypeAny): boolean { export function generateHelpJSON() { const booleanOptions = new Set(); - const commandEntries: Record, args: string[], raw?: boolean }> = {}; + const commandEntries: Record, args: string[], variadicArg?: boolean, raw?: boolean }> = {}; for (const [name, command] of Object.entries(commands)) { const flags: Record = {}; if (command.options) { @@ -177,8 +185,10 @@ export function generateHelpJSON() { booleanOptions.add(flagName); } } - const args: string[] = command.args ? Object.keys((command.args as zodType.ZodObject).shape) : []; - commandEntries[name] = { help: generateCommandHelp(command), flags, args }; + const args = commandArgs(command); + commandEntries[name] = { help: generateCommandHelp(command), flags, args: args.map(a => a.name) }; + if (args.some(a => a.variadic)) + commandEntries[name].variadicArg = true; if (command.raw) commandEntries[name].raw = true; } diff --git a/packages/playwright-core/src/tools/index.ts b/packages/playwright-core/src/tools/index.ts index f0cf3859f9933..5fdad4331b2ac 100644 --- a/packages/playwright-core/src/tools/index.ts +++ b/packages/playwright-core/src/tools/index.ts @@ -30,7 +30,7 @@ export { compareSemver } from './utils/socketConnection'; export { extractTrace, DirTraceLoaderBackend } from './trace/traceParser'; export { decorateMCPCommand } from './mcp/program'; export { program as cliProgram } from './cli-client/program'; -export { generateHelp, generateHelpJSON } from './cli-daemon/helpGenerator'; +export { generateHelp, generateHelpJSON, generateReadme } from './cli-daemon/helpGenerator'; export { decorateProgram as decorateCliDaemonProgram, initWorkspace } from './cli-daemon/program'; export { allSkills, installSkills } from './utils/installSkills'; export { openDashboardApp, openDashboardForContext } from './dashboard/dashboardApp'; diff --git a/packages/playwright/src/runner/lastRun.ts b/packages/playwright/src/runner/lastRun.ts index 57a073f541401..8312da4471ea6 100644 --- a/packages/playwright/src/runner/lastRun.ts +++ b/packages/playwright/src/runner/lastRun.ts @@ -17,7 +17,7 @@ import fs from 'fs'; import path from 'path'; -import type { FullResult, Suite } from '../../types/testReporter'; +import type { FullResult, Suite, TestCase } from '../../types/testReporter'; import type { config as commonConfig } from '../common'; import type { ReporterV2 } from '../reporters/reporterV2'; @@ -26,6 +26,14 @@ type LastRunInfo = { failedTests: string[]; }; +function didNotRun(test: TestCase): boolean { + if (test.outcome() !== 'skipped') + return false; + if (test.results.some(result => result.status === 'interrupted')) + return false; + return !test.results.length || test.expectedStatus !== 'skipped'; +} + export class LastRunReporter implements ReporterV2 { private _lastRunFile: string | undefined; private _suite: Suite | undefined; @@ -71,7 +79,7 @@ export class LastRunReporter implements ReporterV2 { return; const lastRunInfo: LastRunInfo = { status: result.status, - failedTests: this._suite?.allTests().filter(t => !t.ok()).map(t => t.id) || [], + failedTests: this._suite?.allTests().filter(t => !t.ok() || didNotRun(t)).map(t => t.id) || [], }; await fs.promises.mkdir(path.dirname(this._lastRunFile), { recursive: true }); await fs.promises.writeFile(this._lastRunFile, JSON.stringify(lastRunInfo, undefined, 2)); diff --git a/packages/playwright/src/worker/testTracing.ts b/packages/playwright/src/worker/testTracing.ts index e71c2354091e5..3de5f843f6655 100644 --- a/packages/playwright/src/worker/testTracing.ts +++ b/packages/playwright/src/worker/testTracing.ts @@ -172,6 +172,7 @@ export class TestTracing { async stopIfNeeded() { this._contextCreatedEvent.testTimeout = this._testInfo.timeout; + this._contextCreatedEvent.annotations = this._testInfo.annotations.map(({ type, description }) => ({ type, description })); if (!this._options) return; @@ -290,7 +291,7 @@ export class TestTracing { }); } - appendAfterActionForStep(callId: string, error?: trace.SerializedError['error'], attachments: Attachment[] = [], annotations?: trace.AfterActionTraceEventAnnotation[]) { + appendAfterActionForStep(callId: string, error?: trace.SerializedError['error'], attachments: Attachment[] = [], annotations?: trace.TraceEventAnnotation[]) { this._appendTraceEvent({ type: 'after', callId, diff --git a/packages/trace-viewer/src/ui/uiModeTraceView.tsx b/packages/trace-viewer/src/ui/uiModeTraceView.tsx index 31b6c8f9a1e68..30b70a84cba44 100644 --- a/packages/trace-viewer/src/ui/uiModeTraceView.tsx +++ b/packages/trace-viewer/src/ui/uiModeTraceView.tsx @@ -96,7 +96,7 @@ export const TraceView: React.FC<{ fallbackLocation={item.testFile} isLive={model?.isLive} status={item.treeItem?.status} - annotations={item.testCase?.annotations ?? []} + defaultAnnotations={item.testCase?.annotations ?? []} onOpenExternally={onOpenExternally} revealSource={revealSource} />; diff --git a/packages/trace-viewer/src/ui/workbench.tsx b/packages/trace-viewer/src/ui/workbench.tsx index 58ee991e8de45..640f6c69f0fe5 100644 --- a/packages/trace-viewer/src/ui/workbench.tsx +++ b/packages/trace-viewer/src/ui/workbench.tsx @@ -57,7 +57,7 @@ export type WorkbenchProps = { isLive?: boolean; hideTimeline?: boolean; status?: UITestStatus; - annotations?: TestAnnotation[]; + defaultAnnotations?: TestAnnotation[]; inert?: boolean; onOpenExternally?: (location: SourceLocation) => void; revealSource?: boolean; @@ -72,7 +72,9 @@ export const Workbench: React.FunctionComponent = props => { }; const PartitionedWorkbench: React.FunctionComponent = props => { - const { partition, model, showSourcesFirst, rootDir, fallbackLocation, isLive, hideTimeline, status, annotations, inert, onOpenExternally, revealSource, testRunMetadata } = props; + const { partition, model, showSourcesFirst, rootDir, fallbackLocation, isLive, hideTimeline, status, inert, onOpenExternally, revealSource, testRunMetadata } = props; + // Default annotations come from the test model before the test runs, shown for the empty workbench / trace. + const annotations = model?.annotations ?? props.defaultAnnotations; // UI settings, shared for all models. const [selectedNavigatorTab, setSelectedNavigatorTab] = useSetting('navigatorTab', 'actions'); diff --git a/packages/trace/src/trace.ts b/packages/trace/src/trace.ts index e6cd2187765f2..0cb6ae33d1236 100644 --- a/packages/trace/src/trace.ts +++ b/packages/trace/src/trace.ts @@ -94,6 +94,7 @@ export type ContextCreatedTraceEvent = { testIdAttributeName?: string, contextId?: string, testTimeout?: number, + annotations?: TraceEventAnnotation[], }; export type ScreencastFrameTraceEvent = { @@ -137,7 +138,7 @@ export type AfterActionTraceEventAttachment = { base64?: string; }; -export type AfterActionTraceEventAnnotation = { +export type TraceEventAnnotation = { type: string, description?: string }; @@ -149,7 +150,7 @@ export type AfterActionTraceEvent = { afterSnapshot?: string; error?: SerializedError['error']; attachments?: AfterActionTraceEventAttachment[]; - annotations?: AfterActionTraceEventAnnotation[]; + annotations?: TraceEventAnnotation[]; result?: any; point?: Point; }; diff --git a/tests/bidi/expectations/moz-firefox-nightly-library.txt b/tests/bidi/expectations/moz-firefox-nightly-library.txt index 94bf37ef560c1..54d883f479692 100644 --- a/tests/bidi/expectations/moz-firefox-nightly-library.txt +++ b/tests/bidi/expectations/moz-firefox-nightly-library.txt @@ -155,7 +155,6 @@ library/page-event-crash.spec.ts › should cancel navigation when page crashes library/page-event-crash.spec.ts › should cancel waitForEvent when page crashes [timeout] library/page-event-crash.spec.ts › should emit crash event when page crashes [timeout] library/page-event-crash.spec.ts › should throw on any action after page crashes [timeout] -library/permissions.spec.ts › local network request is allowed from public origin [timeout] library/permissions.spec.ts › permissions › should deny permission when not listed [fail] library/permissions.spec.ts › permissions › should fail when bad permission is given [fail] library/permissions.spec.ts › permissions › should isolate permissions between browser contexts [fail] @@ -195,7 +194,8 @@ library/trace-viewer.spec.ts › should pick locator in iframe [fail] library/trace-viewer.spec.ts › should render console [fail] library/trace-viewer.spec.ts › should show only one pointer with multilevel iframes [fail] library/trace-viewer.spec.ts › should show websocket messages [fail] -library/tracing.spec.ts › should produce screencast frames fit [fail] +library/tracing.spec.ts › should produce screencast frames crop [timeout] +library/tracing.spec.ts › should produce screencast frames fit [timeout] library/tracing.spec.ts › should produce screencast frames scale [fail] library/tracing.spec.ts › should save trace while a WebSocket keeps streaming frames [fail] library/web-socket.spec.ts › should emit binary frame events [timeout] diff --git a/tests/bidi/expectations/moz-firefox-nightly-page.txt b/tests/bidi/expectations/moz-firefox-nightly-page.txt index d8cf3f1ca04e8..5fde30e517f8d 100644 --- a/tests/bidi/expectations/moz-firefox-nightly-page.txt +++ b/tests/bidi/expectations/moz-firefox-nightly-page.txt @@ -53,6 +53,8 @@ page/page-accessibility.spec.ts › should work with aria-invalid accessibility page/page-accessibility.spec.ts › should work with regular text [fail] page/page-add-init-script.spec.ts › init script should run only once in iframe [fail] page/page-add-init-script.spec.ts › init script should run only once in popup [timeout] +page/page-add-init-script-callback.spec.ts › should call a function passed as an argument [fail] +page/page-add-init-script-callback.spec.ts › should work in a child frame [fail] page/page-add-script-tag.spec.ts › should throw when added with content to the CSP page [fail] page/page-add-style-tag.spec.ts › should throw when added with content to the CSP page [timeout] page/page-aria-snapshot-ai.spec.ts › return empty snapshot when iframe is not loaded [flaky] @@ -120,6 +122,7 @@ page/page-network-response.spec.ts › should reject response.finished if contex page/page-network-response.spec.ts › should report all headers [fail] page/page-network-response.spec.ts › should report if request was fromServiceWorker [fail] page/page-network-response.spec.ts › should report multiple set-cookie headers [fail] +page/page-network-response.spec.ts › should return non-utf8 body even when content-type says utf8 [fail] page/page-network-response.spec.ts › should wait until response completes [fail] page/page-request-continue.spec.ts › continue with headers should send fresh cookie from the browser cookie store [fail] page/page-request-continue.spec.ts › should not throw if request was cancelled by the page [timeout] diff --git a/tests/library/browsercontext-locale.spec.ts b/tests/library/browsercontext-locale.spec.ts index 28c17871e97cf..bb6ef20563af4 100644 --- a/tests/library/browsercontext-locale.spec.ts +++ b/tests/library/browsercontext-locale.spec.ts @@ -225,8 +225,8 @@ it('should send user Accept-Language header', { it('should send Accept-Language header on WebSocket handshake', { annotation: [{ type: 'issue', description: 'https://github.com/microsoft/playwright/issues/23732' }], -}, async ({ browser, server, browserName, browserMajorVersion }) => { - it.fixme(browserName === 'firefox', 'Firefox does not send Accept-Language on WebSocket handshake'); +}, async ({ browser, server, browserName, browserMajorVersion, isBidi }) => { + it.fixme(browserName === 'firefox' && !isBidi, 'Firefox/Juggler does not send Accept-Language on WebSocket handshake'); it.fixme(browserName === 'chromium' && browserMajorVersion === 150, 'Chromium 150 sends the browser Accept-Language instead of the emulated locale on WebSocket handshake, https://github.com/microsoft/playwright/issues/23732'); const context = await browser.newContext({ locale: 'en-GB' }); const page = await context.newPage(); diff --git a/tests/library/client-certificates.spec.ts b/tests/library/client-certificates.spec.ts index a468dd48bb1fa..0684a504d3063 100644 --- a/tests/library/client-certificates.spec.ts +++ b/tests/library/client-certificates.spec.ts @@ -344,7 +344,7 @@ test.describe('browser', () => { test('should not intercept TLS for origins without a client certificate', { annotation: { type: 'issue', description: 'https://github.com/microsoft/playwright/issues/41106' }, - }, async ({ browser, asset, httpsServer, browserName, platform, channel }) => { + }, async ({ browser, asset, httpsServer }) => { // If the proxy intercepted this origin, the browser would see its self-signed cert (CN=localhost) // instead of the real server cert (CN=playwright-test). const page = await browser.newPage({ @@ -356,14 +356,8 @@ test.describe('browser', () => { }); const response = await page.goto(httpsServer.EMPTY_PAGE); expect(response.ok()).toBe(true); - const subjectName = (await response.securityDetails()).subjectName; - if (browserName === 'webkit' && platform === 'win32' && channel !== 'webkit-wsl') { - // Don't ask me why this is "true" on Windows WebKit. - expect(subjectName).toContain('true'); - } else { - // This is "CN=playwright-test" in some ubuntu webkits, and "playwright-test" in other browsers. - expect(subjectName).toContain('playwright-test'); - } + const securityDetails = await response.securityDetails(); + expect(securityDetails.subjectName).toContain('playwright-test'); await page.close(); }); diff --git a/tests/library/har.spec.ts b/tests/library/har.spec.ts index 984091ed7b0cc..143a3186b1dc0 100644 --- a/tests/library/har.spec.ts +++ b/tests/library/har.spec.ts @@ -697,18 +697,16 @@ it('should return server address directly from response', async ({ page, server, expect(port).toBe(server.PORT); }); -it('should return security details directly from response', async ({ contextFactory, httpsServer, browserName, platform, channel, isFrozenWebkit }) => { +it('should return security details directly from response', async ({ contextFactory, httpsServer, browserName, platform, isFrozenWebkit }) => { it.skip(isFrozenWebkit); const context = await contextFactory({ ignoreHTTPSErrors: true }); const page = await context.newPage(); const response = await page.goto(httpsServer.EMPTY_PAGE); const securityDetails = await response!.securityDetails(); - if (channel === 'webkit-wsl') - // The Linux WebKit build reports the real subject name, but (like the Windows port) does not surface the TLS protocol. + if (browserName === 'webkit' && platform === 'win32') + // WebKit on Windows does not surface the TLS protocol. expect({ ...securityDetails, protocol: undefined }).toEqual({ subjectName: 'playwright-test', validFrom: 1691708270, validTo: 2007068270 }); - else if (browserName === 'webkit' && platform === 'win32') - expect({ ...securityDetails, protocol: undefined }).toEqual({ subjectName: 'true', validFrom: 1691708270, validTo: 2007068270 }); else if (browserName === 'webkit') expect(securityDetails).toEqual({ protocol: 'TLS 1.3', subjectName: 'playwright-test', validFrom: 1691708270, validTo: 2007068270 }); else diff --git a/tests/mcp/cli-core.spec.ts b/tests/mcp/cli-core.spec.ts index 4b3cd8cecd81e..00dc045122414 100644 --- a/tests/mcp/cli-core.spec.ts +++ b/tests/mcp/cli-core.spec.ts @@ -146,6 +146,33 @@ test('uncheck', async ({ cli, server, mcpBrowser }) => { expect(inlineSnapshot).toContain(`- checkbox ${active}[ref=e2]`); }); +test('upload multiple files', { annotation: { type: 'issue', description: 'https://github.com/microsoft/playwright/issues/42047' } }, async ({ cli, server }, testInfo) => { + server.setContent('/', ` + +
+ + `, 'text/html'); + + const front = testInfo.outputPath('front.txt'); + const back = testInfo.outputPath('back.txt'); + await fs.promises.writeFile(front, 'front'); + await fs.promises.writeFile(back, 'back'); + + await cli('open', server.PREFIX); + await cli('click', 'e2'); + const { output, snapshot } = await cli('upload', front, back); + expect(output).toContain('await fileChooser.setFiles('); + expect(snapshot).toContain('Received: front.txt, back.txt'); + + await cli('click', 'e2'); + const single = await cli('upload', back); + expect(single.snapshot).toContain('Received: back.txt'); +}); + test('eval', async ({ cli, server }) => { await cli('open', server.HELLO_WORLD); const { output } = await cli('eval', '() => document.title'); @@ -369,6 +396,18 @@ test('--raw on command without output', async ({ cli, server }) => { expect(output).not.toContain('Page URL'); }); +test('tool error exits with non-zero code', { annotation: { type: 'issue', description: 'https://github.com/microsoft/playwright/issues/42028' } }, async ({ cli, server }) => { + await cli('open', server.HELLO_WORLD); + + const { output, exitCode } = await cli('click', 'e999'); + expect(output).toContain('Ref e999 not found in the current page snapshot.'); + expect(exitCode).toBe(1); + + const { output: jsonOutput, exitCode: jsonExitCode } = await cli('--json', 'click', 'e999'); + expect(JSON.parse(jsonOutput).isError).toBe(true); + expect(jsonExitCode).toBe(1); +}); + test('codegen escapes single quotes in user input', async ({ cli, server }) => { server.setContent('/', ``, 'text/html'); await cli('open', server.PREFIX); diff --git a/tests/mcp/cli-help.spec.ts b/tests/mcp/cli-help.spec.ts index 0b3150c82f671..be25ffbe13296 100644 --- a/tests/mcp/cli-help.spec.ts +++ b/tests/mcp/cli-help.spec.ts @@ -31,6 +31,11 @@ test('prints command help', async ({ cli }) => { expect(output).toContain('playwright-cli click [button]'); }); +test('prints variadic command help', { annotation: { type: 'issue', description: 'https://github.com/microsoft/playwright/issues/42047' } }, async ({ cli }) => { + const { output } = await cli('upload', '--help'); + expect(output).toContain('playwright-cli upload '); +}); + test('prints agent skill path when running under a coding agent', async ({ cli }) => { const { output } = await cli('--help', { env: { CLAUDECODE: '1' } }); expect(output).toContain('Agent skill:'); diff --git a/tests/mcp/cli-parsing.spec.ts b/tests/mcp/cli-parsing.spec.ts index 1c9957b9436bc..8b65fbcba86c8 100644 --- a/tests/mcp/cli-parsing.spec.ts +++ b/tests/mcp/cli-parsing.spec.ts @@ -57,6 +57,13 @@ test('missing argument', async ({ cli, server }) => { expect(error).toContain(`error: 'key' argument: expected string, received undefined`); }); +test('missing variadic argument', { annotation: { type: 'issue', description: 'https://github.com/microsoft/playwright/issues/42047' } }, async ({ cli, server }) => { + await cli('open', server.HELLO_WORLD); + const { error, exitCode } = await cli('upload'); + expect(exitCode).toBe(1); + expect(error).toContain(`error: 'files' argument: expected string, received undefined`); +}); + test('wrong argument type', async ({ cli, server }) => { await cli('open', server.HELLO_WORLD); const { error, exitCode } = await cli('mousemove', '12', 'foo'); diff --git a/tests/page/page-goto.spec.ts b/tests/page/page-goto.spec.ts index 90c0ad8930d08..c096d1529915f 100644 --- a/tests/page/page-goto.spec.ts +++ b/tests/page/page-goto.spec.ts @@ -34,6 +34,15 @@ it('should work with file URL', async ({ page, asset, isAndroid, mode, channel } expect(page.frames().length).toBe(1); }); +it('should navigate from file URL to about:blank', async ({ page, asset, isAndroid, channel }) => { + it.info().annotations.push({ type: 'issue', description: 'https://github.com/microsoft/playwright/issues/42050' }); + it.skip(isAndroid, 'No files on Android'); + it.skip(channel === 'webkit-wsl', 'separate filesystem on wsl'); + + await page.goto(url.pathToFileURL(asset('empty.html')).href); + await page.goto('about:blank'); +}); + it('should work with file URL with subframes', async ({ page, asset, isAndroid, mode, channel }) => { it.skip(isAndroid, 'No files on Android'); it.skip(channel === 'webkit-wsl', 'separate filesystem on wsl'); diff --git a/tests/playwright-test/playwright.trace.spec.ts b/tests/playwright-test/playwright.trace.spec.ts index 78ebd785be167..810be371d828a 100644 --- a/tests/playwright-test/playwright.trace.spec.ts +++ b/tests/playwright-test/playwright.trace.spec.ts @@ -1427,3 +1427,28 @@ test('should record custom test timeout in trace', async ({ runInlineTest }, tes const trace = await parseTrace(testInfo.outputPath('test-results', 'a-pass', 'trace.zip')); expect(trace.model.testTimeout).toBe(120_000); }); + +test('should record test annotations in trace', async ({ runInlineTest }, testInfo) => { + test.info().annotations.push({ type: 'issue', description: 'https://github.com/microsoft/playwright/issues/42035' }); + + const result = await runInlineTest({ + 'a.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('pass', { + annotation: { type: 'note1', description: 'static annotation' }, + }, async ({}) => { + test.info().annotations.push({ type: 'note2', description: 'dynamic annotation' }); + test.info().annotations.push({ type: 'note3' }); + }); + `, + }, { trace: 'on' }); + + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(1); + const trace = await parseTrace(testInfo.outputPath('test-results', 'a-pass', 'trace.zip')); + expect(trace.model.annotations).toEqual([ + { type: 'note1', description: 'static annotation' }, + { type: 'note2', description: 'dynamic annotation' }, + { type: 'note3' }, + ]); +}); diff --git a/tests/playwright-test/runner.spec.ts b/tests/playwright-test/runner.spec.ts index 4973103484383..812f6cade29ed 100644 --- a/tests/playwright-test/runner.spec.ts +++ b/tests/playwright-test/runner.spec.ts @@ -866,6 +866,55 @@ test('should run nothing with --last-failed when previous run had no failures', expect(result2.didNotRun).toBe(0); }); +test('should run last failed tests that did not run because a beforeAll hook failed', async ({ runInlineTest }) => { + const workspace = { + 'a.spec.js': ` + import fs from 'fs'; + import { test, expect } from '@playwright/test'; + test.beforeAll(() => { + if (!fs.existsSync('marker.txt')) { + fs.writeFileSync('marker.txt', ''); + throw new Error('from beforeAll'); + } + }); + test('one', () => {}); + test('two', () => {}); + ` + }; + const result1 = await runInlineTest(workspace); + expect(result1.exitCode).toBe(1); + expect(result1.failed).toBe(1); + expect(result1.didNotRun).toBe(1); + + const result2 = await runInlineTest(workspace, {}, {}, { additionalArgs: ['--last-failed'] }); + expect(result2.exitCode).toBe(0); + expect(result2.passed).toBe(2); + expect(result2.didNotRun).toBe(0); +}); + +test('should not run intentionally skipped tests with --last-failed', async ({ runInlineTest }) => { + const workspace = { + 'a.spec.js': ` + import { test, expect } from '@playwright/test'; + test('fail', () => { + expect(1).toBe(2); + }); + test('skipped', () => { + test.skip(); + }); + ` + }; + const result1 = await runInlineTest(workspace); + expect(result1.exitCode).toBe(1); + expect(result1.failed).toBe(1); + expect(result1.skipped).toBe(1); + + const result2 = await runInlineTest(workspace, {}, {}, { additionalArgs: ['--last-failed'] }); + expect(result2.exitCode).toBe(1); + expect(result2.failed).toBe(1); + expect(result2.skipped).toBe(0); +}); + test('should run last failed tests in a shard', async ({ runInlineTest }) => { const workspace = { 'a.spec.js': ` diff --git a/utils/build/build.js b/utils/build/build.js index 2bef516d944d4..466988323453b 100644 --- a/utils/build/build.js +++ b/utils/build/build.js @@ -946,6 +946,7 @@ steps.push(new ProgramStep({ // Generate CLI help. onChanges.push({ inputs: [ + 'packages/playwright-core/src/tools/cli-daemon/command.ts', 'packages/playwright-core/src/tools/cli-daemon/commands.ts', 'packages/playwright-core/src/tools/cli-daemon/helpGenerator.ts', 'utils/generate_cli_help.js',