Skip to content

[pull] main from microsoft:main #812

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
pull merged 6 commits into from
Jul 3, 2026
Merged

[pull] main from microsoft:main #812

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
6680549
feat(cli): add init-skills command to install the Playwright CLI skil…
pavelfeldman Jul 2, 2026
3343395
chore(aria): remove unused incremental ariaSnapshot track option (#41…
pavelfeldman Jul 2, 2026
bd390fa
test(webkit): unskip right-click tests fixed by browser roll (#41602)
yury-s Jul 2, 2026
1cc3b5c
docs(cli): unify test-generation and spec-driven-testing skills (#41603)
pavelfeldman Jul 2, 2026
26385ab
feat(aria): distill ai snapshots to make them less verbose (#41604)
pavelfeldman Jul 3, 2026
9725152
feat(mcp): add browser_find tool to search page snapshot (#41605)
pavelfeldman Jul 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
195 changes: 35 additions & 160 deletions packages/injected/src/ariaSnapshot.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,13 @@ import * as aria from '@isomorphic/ariaSnapshot';
import { escapeRegExp, longestCommonSubstring, normalizeWhiteSpace, truncateDataUrl } from '@isomorphic/stringUtils';
import { yamlEscapeKeyIfNeeded, yamlEscapeValueIfNeeded } from '@isomorphic/yaml';

import { distillAriaSnapshot } from './ariaSnapshotDistiller';
import { computeBox, getElementComputedStyle, isElementVisible } from './domUtils';
import * as roleUtils from './roleUtils';

export type AriaSnapshot = {
root: aria.AriaNode;
elements: Map<string, Element>;
info: Map<string, { element: Element, nameFromContentRefs: string[] }>;
refs: Map<Element, string>;
iframeRefs: string[];
};
Expand Down Expand Up @@ -84,10 +85,12 @@ function toInternalOptions(options: AriaTreeOptions): InternalOptions {
export function generateAriaTree(rootElement: Element, publicOptions: AriaTreeOptions): AriaSnapshot {
const options = toInternalOptions(publicOptions);
const visited = new Set<Node>();
// For each node, the elements that contributed to its accessible name.
const nameSourceElements = new Map<aria.AriaNode, Set<Element> | undefined>();

const snapshot: AriaSnapshot = {
root: { role: 'fragment', name: '', children: [], props: {}, box: computeBox(rootElement), receivesPointerEvents: true },
elements: new Map<string, Element>(),
info: new Map<string, { element: Element, nameFromContentRefs: string[] }>(),
refs: new Map<Element, string>(),
iframeRefs: [],
};
Expand Down Expand Up @@ -135,17 +138,29 @@ export function generateAriaTree(rootElement: Element, publicOptions: AriaTreeOp
}
}

const childAriaNode = visible ? toAriaNode(element, options) : null;
const childAriaNode = visible ? toAriaNode(element, options, nameSourceElements) : null;
let elementInfo: { element: Element, nameFromContentRefs: string[] } | undefined;
if (childAriaNode) {
if (childAriaNode.ref) {
snapshot.elements.set(childAriaNode.ref, element);
elementInfo = { element, nameFromContentRefs: [] };
snapshot.info.set(childAriaNode.ref, elementInfo);
snapshot.refs.set(element, childAriaNode.ref);
if (childAriaNode.role === 'iframe')
snapshot.iframeRefs.push(childAriaNode.ref);
}
ariaNode.children.push(childAriaNode);
}
processElement(childAriaNode || ariaNode, element, ariaChildren, visible);

// Now that the subtree is processed, every descendant that contributed to this node's
// accessible name has its ref assigned, so we can resolve those refs as the name's origins.
if (elementInfo) {
for (const contributor of nameSourceElements.get(childAriaNode!) || []) {
const ref = snapshot.refs.get(contributor);
if (ref && ref !== childAriaNode!.ref)
elementInfo.nameFromContentRefs.push(ref);
}
}
};

function processElement(ariaNode: aria.AriaNode, element: Element, ariaChildren: Element[], parentElementVisible: boolean) {
Expand Down Expand Up @@ -200,8 +215,7 @@ export function generateAriaTree(rootElement: Element, publicOptions: AriaTreeOp
roleUtils.endAriaCaches();
}

normalizeStringChildren(snapshot.root);
normalizeGenericRoles(snapshot.root);
distillAriaSnapshot(snapshot, publicOptions);
return snapshot;
}

Expand All @@ -220,8 +234,8 @@ function computeAriaRef(ariaNode: aria.AriaNode, options: InternalOptions) {
ariaNode.ref = ariaRef.ref;
}

function toAriaNode(element: Element, options: InternalOptions): aria.AriaNode | null {
const active = element.ownerDocument.activeElement === element;
function toAriaNode(element: Element, options: InternalOptions, nameSourceElements: Map<aria.AriaNode, Set<Element> | undefined>): aria.AriaNode | null {
const active = element.ownerDocument.activeElement === element && element.ownerDocument.hasFocus();
if (element.nodeName === 'IFRAME') {
const ariaNode: aria.AriaNode = {
role: 'iframe',
Expand All @@ -242,7 +256,7 @@ function toAriaNode(element: Element, options: InternalOptions): aria.AriaNode |
if (!role || role === 'presentation' || role === 'none')
return null;

const name = normalizeWhiteSpace(roleUtils.getElementAccessibleName(element, false) || '');
const name = roleUtils.getElementAccessibleName(element, false);
const receivesPointerEvents = roleUtils.receivesPointerEvents(element);

const box = computeBox(element);
Expand All @@ -251,14 +265,15 @@ function toAriaNode(element: Element, options: InternalOptions): aria.AriaNode |

const result: aria.AriaNode = {
role,
name,
name: normalizeWhiteSpace(name.text),
children: [],
props: {},
box,
receivesPointerEvents,
active
};
setAriaNodeElement(result, element);
nameSourceElements.set(result, name.elements);
computeAriaRef(result, options);

if (roleUtils.kAriaCheckedRoles.includes(role))
Expand Down Expand Up @@ -292,59 +307,6 @@ function toAriaNode(element: Element, options: InternalOptions): aria.AriaNode |
return result;
}

function normalizeGenericRoles(node: aria.AriaNode) {
const normalizeChildren = (node: aria.AriaNode) => {
const result: (aria.AriaNode | string)[] = [];
for (const child of node.children || []) {
if (typeof child === 'string') {
result.push(child);
continue;
}
const normalized = normalizeChildren(child);
result.push(...normalized);
}

// Only remove generic that encloses one element, logical grouping still makes sense, even if it is not ref-able.
const removeSelf = node.role === 'generic' && !node.name && result.length <= 1 && result.every(c => typeof c !== 'string' && !!c.ref);
if (removeSelf)
return result;
node.children = result;
return [node];
};

normalizeChildren(node);
}

function normalizeStringChildren(rootA11yNode: aria.AriaNode) {
const flushChildren = (buffer: string[], normalizedChildren: (aria.AriaNode | string)[]) => {
if (!buffer.length)
return;
const text = normalizeWhiteSpace(buffer.join(''));
if (text)
normalizedChildren.push(text);
buffer.length = 0;
};

const visit = (ariaNode: aria.AriaNode) => {
const normalizedChildren: (aria.AriaNode | string)[] = [];
const buffer: string[] = [];
for (const child of ariaNode.children || []) {
if (typeof child === 'string') {
buffer.push(child);
} else {
flushChildren(buffer, normalizedChildren);
visit(child);
normalizedChildren.push(child);
}
}
flushChildren(buffer, normalizedChildren);
ariaNode.children = normalizedChildren.length ? normalizedChildren : [];
if (ariaNode.children.length === 1 && ariaNode.children[0] === ariaNode.name)
ariaNode.children = [];
};
visit(rootA11yNode);
}

function matchesStringOrRegex(text: string, template: aria.AriaRegex | string | undefined): boolean {
if (!template)
return true;
Expand Down Expand Up @@ -500,98 +462,19 @@ function matchesNodeDeep(root: aria.AriaNode, template: aria.AriaTemplateNode, c
return results;
}

function buildByRefMap(root: aria.AriaNode | undefined, map: Map<string | undefined, aria.AriaNode> = new Map()): Map<string | undefined, aria.AriaNode> {
if (root?.ref)
map.set(root.ref, root);
for (const child of root?.children || []) {
if (typeof child !== 'string')
buildByRefMap(child, map);
}
return map;
}

function compareSnapshots(ariaSnapshot: AriaSnapshot, previousSnapshot: AriaSnapshot | undefined): Map<aria.AriaNode, 'skip' | 'same' | 'changed'> {
const previousByRef = buildByRefMap(previousSnapshot?.root);
const result = new Map<aria.AriaNode, 'skip' | 'same' | 'changed'>();

// Returns whether ariaNode is the same as previousNode.
const visit = (ariaNode: aria.AriaNode, previousNode: aria.AriaNode | undefined): boolean => {
let same: boolean = ariaNode.children.length === previousNode?.children.length && aria.ariaNodesEqual(ariaNode, previousNode);
let canBeSkipped = same;

for (let childIndex = 0 ; childIndex < ariaNode.children.length; childIndex++) {
const child = ariaNode.children[childIndex];
const previousChild = previousNode?.children[childIndex];
if (typeof child === 'string') {
same &&= child === previousChild;
canBeSkipped &&= child === previousChild;
} else {
let previous = typeof previousChild !== 'string' ? previousChild : undefined;
if (child.ref)
previous = previousByRef.get(child.ref);
const sameChild = visit(child, previous);
// New child, different order of children, or changed child with no ref -
// we have to include this node to list children in the right order.
if (!previous || (!sameChild && !child.ref) || (previous !== previousChild))
canBeSkipped = false;
same &&= (sameChild && previous === previousChild);
}
}

result.set(ariaNode, same ? 'same' : (canBeSkipped ? 'skip' : 'changed'));
return same;
};

visit(ariaSnapshot.root, previousByRef.get(previousSnapshot?.root?.ref));
return result;
}

// Chooses only the changed parts of the snapshot and returns them as new roots.
function filterSnapshotDiff(nodes: (aria.AriaNode | string)[], statusMap: Map<aria.AriaNode, 'skip' | 'same' | 'changed'>): (aria.AriaNode | string)[] {
const result: (aria.AriaNode | string)[] = [];

const visit = (ariaNode: aria.AriaNode) => {
const status = statusMap.get(ariaNode);
if (status === 'same') {
// No need to render unchanged root at all.
} else if (status === 'skip') {
// Only render changed children.
for (const child of ariaNode.children) {
if (typeof child !== 'string')
visit(child);
}
} else {
// Render this node's subtree.
result.push(ariaNode);
}
};

for (const node of nodes) {
if (typeof node === 'string')
result.push(node);
else
visit(node);
}
return result;
}

function indent(depth: number): string {
return ' '.repeat(depth);
}

export function renderAriaTree(ariaSnapshot: AriaSnapshot, publicOptions: AriaTreeOptions, previousSnapshot?: AriaSnapshot): { text: string, iframeDepths: Record<string, number> } {
export function renderAriaTree(ariaSnapshot: AriaSnapshot, publicOptions: AriaTreeOptions): { text: string, iframeDepths: Record<string, number> } {
const options = toInternalOptions(publicOptions);
const lines: string[] = [];
const iframeDepths: Record<string, number> = {};
const includeText = options.renderStringsAsRegex ? textContributesInfo : () => true;
const renderString = options.renderStringsAsRegex ? convertToBestGuessRegex : (str: string) => str;

// Do not render the root fragment, just its children.
let nodesToRender = ariaSnapshot.root.role === 'fragment' ? ariaSnapshot.root.children : [ariaSnapshot.root];

const statusMap = compareSnapshots(ariaSnapshot, previousSnapshot);
if (previousSnapshot)
nodesToRender = filterSnapshotDiff(nodesToRender, statusMap);
const nodesToRender = ariaSnapshot.root.role === 'fragment' ? ariaSnapshot.root.children : [ariaSnapshot.root];

const visitText = (text: string, depth: number) => {
if (publicOptions.depth && depth > publicOptions.depth)
Expand Down Expand Up @@ -649,8 +532,8 @@ export function renderAriaTree(ariaSnapshot: AriaSnapshot, publicOptions: AriaTr
return key;
};

const getSingleInlinedTextChild = (ariaNode: aria.AriaNode | undefined): string | undefined => {
return ariaNode?.children.length === 1 && typeof ariaNode.children[0] === 'string' && !Object.keys(ariaNode.props).length ? ariaNode.children[0] : undefined;
const getSingleTextChild = (ariaNode: aria.AriaNode): string | undefined => {
return ariaNode.children.length === 1 && typeof ariaNode.children[0] === 'string' && !Object.keys(ariaNode.props).length ? ariaNode.children[0] : undefined;
};

const visit = (ariaNode: aria.AriaNode, depth: number, renderCursorPointer: boolean) => {
Expand All @@ -660,27 +543,19 @@ export function renderAriaTree(ariaSnapshot: AriaSnapshot, publicOptions: AriaTr
if (ariaNode.role === 'iframe' && ariaNode.ref)
iframeDepths[ariaNode.ref] = depth;

// Replace the whole subtree with a single reference when possible.
if (statusMap.get(ariaNode) === 'same' && ariaNode.ref) {
lines.push(indent(depth) + `- ref=${ariaNode.ref} [unchanged]`);
return;
}

// When producing a diff, add <changed> marker to all diff roots.
const isDiffRoot = !!previousSnapshot && !depth;
const escapedKey = indent(depth) + '- ' + (isDiffRoot ? '<changed> ' : '') + yamlEscapeKeyIfNeeded(createKey(ariaNode, renderCursorPointer));
const singleInlinedTextChild = getSingleInlinedTextChild(ariaNode);
const escapedKey = indent(depth) + '- ' + yamlEscapeKeyIfNeeded(createKey(ariaNode, renderCursorPointer));
const singleTextChild = getSingleTextChild(ariaNode);
const isAtDepthLimit = !!publicOptions.depth && depth === publicOptions.depth;
const hasNoChildren = !singleInlinedTextChild && (!ariaNode.children.length || isAtDepthLimit);
const hasNoChildren = !singleTextChild && (!ariaNode.children.length || isAtDepthLimit);

if (hasNoChildren && !Object.keys(ariaNode.props).length) {
// Leaf node without children.
lines.push(escapedKey);
} else if (singleInlinedTextChild !== undefined) {
} else if (singleTextChild !== undefined) {
// Leaf node with just some text inside.
const shouldInclude = includeText(ariaNode, singleInlinedTextChild);
const shouldInclude = includeText(ariaNode, singleTextChild);
if (shouldInclude)
lines.push(escapedKey + ': ' + yamlEscapeValueIfNeeded(renderString(singleInlinedTextChild)));
lines.push(escapedKey + ': ' + yamlEscapeValueIfNeeded(renderString(singleTextChild)));
else
lines.push(escapedKey);
} else {
Expand Down
Loading
Loading