From 0057f3fe0a8d179cacb18495c96987310f83d5d9 Mon Sep 17 00:00:00 2001 From: Alex Rudenko Date: Wed, 12 Jun 2024 10:40:36 +0200 Subject: [PATCH] feat: allow creating ElementHandles from the accessibility tree snapshot (#12233) --- ...uppeteer.serializedaxnode.elementhandle.md | 21 +++++++ docs/api/puppeteer.serializedaxnode.md | 24 ++++++++ packages/puppeteer-core/src/api/Frame.ts | 6 ++ packages/puppeteer-core/src/api/Page.ts | 4 +- packages/puppeteer-core/src/bidi/Frame.ts | 3 + packages/puppeteer-core/src/bidi/Page.ts | 3 - .../puppeteer-core/src/cdp/Accessibility.ts | 58 ++++++++++++------- packages/puppeteer-core/src/cdp/Frame.ts | 4 ++ packages/puppeteer-core/src/cdp/Page.ts | 8 --- test/src/accessibility.spec.ts | 46 +++++++++++---- 10 files changed, 134 insertions(+), 43 deletions(-) create mode 100644 docs/api/puppeteer.serializedaxnode.elementhandle.md diff --git a/docs/api/puppeteer.serializedaxnode.elementhandle.md b/docs/api/puppeteer.serializedaxnode.elementhandle.md new file mode 100644 index 00000000000..a4e5bf3171b --- /dev/null +++ b/docs/api/puppeteer.serializedaxnode.elementhandle.md @@ -0,0 +1,21 @@ +--- +sidebar_label: SerializedAXNode.elementHandle +--- + +# SerializedAXNode.elementHandle() method + +Get an ElementHandle for this AXNode if available. + +If the underlying DOM element has been disposed, the method might return an error. + +#### Signature: + +```typescript +interface SerializedAXNode { + elementHandle(): Promise; +} +``` + +**Returns:** + +Promise<[ElementHandle](./puppeteer.elementhandle.md) \| null> diff --git a/docs/api/puppeteer.serializedaxnode.md b/docs/api/puppeteer.serializedaxnode.md index acc12ed78ad..ac2e1014278 100644 --- a/docs/api/puppeteer.serializedaxnode.md +++ b/docs/api/puppeteer.serializedaxnode.md @@ -502,3 +502,27 @@ A description of the current value. + +## Methods + + + +
+ +Method + + + +Description + +
+ +[elementHandle()](./puppeteer.serializedaxnode.elementhandle.md) + + + +Get an ElementHandle for this AXNode if available. + +If the underlying DOM element has been disposed, the method might return an error. + +
diff --git a/packages/puppeteer-core/src/api/Frame.ts b/packages/puppeteer-core/src/api/Frame.ts index 659b2f2919e..8600e556b98 100644 --- a/packages/puppeteer-core/src/api/Frame.ts +++ b/packages/puppeteer-core/src/api/Frame.ts @@ -14,6 +14,7 @@ import type { WaitForSelectorOptions, WaitTimeoutOptions, } from '../api/Page.js'; +import type {Accessibility} from '../cdp/Accessibility.js'; import type {DeviceRequestPrompt} from '../cdp/DeviceRequestPrompt.js'; import type {PuppeteerLifeCycleEvent} from '../cdp/LifecycleWatcher.js'; import {EventEmitter, type EventType} from '../common/EventEmitter.js'; @@ -379,6 +380,11 @@ export abstract class Frame extends EventEmitter { */ abstract get client(): CDPSession; + /** + * @internal + */ + abstract get accessibility(): Accessibility; + /** * @internal */ diff --git a/packages/puppeteer-core/src/api/Page.ts b/packages/puppeteer-core/src/api/Page.ts index ef1ff9b23de..930a9d71823 100644 --- a/packages/puppeteer-core/src/api/Page.ts +++ b/packages/puppeteer-core/src/api/Page.ts @@ -843,7 +843,9 @@ export abstract class Page extends EventEmitter { /** * {@inheritDoc Accessibility} */ - abstract get accessibility(): Accessibility; + get accessibility(): Accessibility { + return this.mainFrame().accessibility; + } /** * An array of all frames attached to the page. diff --git a/packages/puppeteer-core/src/bidi/Frame.ts b/packages/puppeteer-core/src/bidi/Frame.ts index 1fde5c2b805..3f4414a662f 100644 --- a/packages/puppeteer-core/src/bidi/Frame.ts +++ b/packages/puppeteer-core/src/bidi/Frame.ts @@ -27,6 +27,7 @@ import { type WaitForOptions, } from '../api/Frame.js'; import {PageEvent} from '../api/Page.js'; +import {Accessibility} from '../cdp/Accessibility.js'; import { ConsoleMessage, type ConsoleMessageLocation, @@ -71,6 +72,7 @@ export class BidiFrame extends Frame { override readonly _id: string; override readonly client: BidiCdpSession; + override readonly accessibility: Accessibility; private constructor( parent: BidiPage | BidiFrame, @@ -91,6 +93,7 @@ export class BidiFrame extends Frame { this ), }; + this.accessibility = new Accessibility(this.realms.default); } #initialize(): void { diff --git a/packages/puppeteer-core/src/bidi/Page.ts b/packages/puppeteer-core/src/bidi/Page.ts index e121350ec59..b24791a0eba 100644 --- a/packages/puppeteer-core/src/bidi/Page.ts +++ b/packages/puppeteer-core/src/bidi/Page.ts @@ -24,7 +24,6 @@ import { type NewDocumentScriptEvaluation, type ScreenshotOptions, } from '../api/Page.js'; -import {Accessibility} from '../cdp/Accessibility.js'; import {Coverage} from '../cdp/Coverage.js'; import {EmulationManager} from '../cdp/EmulationManager.js'; import {Tracing} from '../cdp/Tracing.js'; @@ -86,7 +85,6 @@ export class BidiPage extends Page { readonly keyboard: BidiKeyboard; readonly mouse: BidiMouse; readonly touchscreen: BidiTouchscreen; - readonly accessibility: Accessibility; readonly tracing: Tracing; readonly coverage: Coverage; readonly #cdpEmulationManager: EmulationManager; @@ -104,7 +102,6 @@ export class BidiPage extends Page { this.#frame = BidiFrame.from(this, browsingContext); this.#cdpEmulationManager = new EmulationManager(this.#frame.client); - this.accessibility = new Accessibility(this.#frame.client); this.tracing = new Tracing(this.#frame.client); this.coverage = new Coverage(this.#frame.client); this.keyboard = new BidiKeyboard(this); diff --git a/packages/puppeteer-core/src/cdp/Accessibility.ts b/packages/puppeteer-core/src/cdp/Accessibility.ts index d0279e3ddad..d79a5707669 100644 --- a/packages/puppeteer-core/src/cdp/Accessibility.ts +++ b/packages/puppeteer-core/src/cdp/Accessibility.ts @@ -6,8 +6,8 @@ import type {Protocol} from 'devtools-protocol'; -import type {CDPSession} from '../api/CDPSession.js'; import type {ElementHandle} from '../api/ElementHandle.js'; +import type {Realm} from '../api/Realm.js'; /** * Represents a Node and the properties of it that are relevant to Accessibility. @@ -80,6 +80,14 @@ export interface SerializedAXNode { * Children of this node, if there are any. */ children?: SerializedAXNode[]; + + /** + * Get an ElementHandle for this AXNode if available. + * + * If the underlying DOM element has been disposed, the method might return an + * error. + */ + elementHandle(): Promise; } /** @@ -121,20 +129,13 @@ export interface SnapshotOptions { * @public */ export class Accessibility { - #client: CDPSession; + #realm: Realm; /** * @internal */ - constructor(client: CDPSession) { - this.#client = client; - } - - /** - * @internal - */ - updateClient(client: CDPSession): void { - this.#client = client; + constructor(realm: Realm) { + this.#realm = realm; } /** @@ -180,15 +181,20 @@ export class Accessibility { options: SnapshotOptions = {} ): Promise { const {interestingOnly = true, root = null} = options; - const {nodes} = await this.#client.send('Accessibility.getFullAXTree'); + const {nodes} = await this.#realm.environment.client.send( + 'Accessibility.getFullAXTree' + ); let backendNodeId: number | undefined; if (root) { - const {node} = await this.#client.send('DOM.describeNode', { - objectId: root.id, - }); + const {node} = await this.#realm.environment.client.send( + 'DOM.describeNode', + { + objectId: root.id, + } + ); backendNodeId = node.backendNodeId; } - const defaultRoot = AXNode.createTree(nodes); + const defaultRoot = AXNode.createTree(this.#realm, nodes); let needle: AXNode | null = defaultRoot; if (backendNodeId) { needle = defaultRoot.find(node => { @@ -260,13 +266,14 @@ class AXNode { #role: string; #ignored: boolean; #cachedHasFocusableChild?: boolean; + #realm: Realm; - constructor(payload: Protocol.Accessibility.AXNode) { + constructor(realm: Realm, payload: Protocol.Accessibility.AXNode) { this.payload = payload; this.#name = this.payload.name ? this.payload.name.value : ''; this.#role = this.payload.role ? this.payload.role.value : 'Unknown'; this.#ignored = this.payload.ignored; - + this.#realm = realm; for (const property of this.payload.properties || []) { if (property.name === 'editable') { this.#richlyEditable = property.value.value === 'richtext'; @@ -441,6 +448,14 @@ class AXNode { const node: SerializedAXNode = { role: this.#role, + elementHandle: async (): Promise => { + if (!this.payload.backendDOMNodeId) { + return null; + } + return (await this.#realm.adoptBackendNode( + this.payload.backendDOMNodeId + )) as ElementHandle; + }, }; type UserStringProperty = @@ -561,10 +576,13 @@ class AXNode { return node; } - public static createTree(payloads: Protocol.Accessibility.AXNode[]): AXNode { + public static createTree( + realm: Realm, + payloads: Protocol.Accessibility.AXNode[] + ): AXNode { const nodeById = new Map(); for (const payload of payloads) { - nodeById.set(payload.nodeId, new AXNode(payload)); + nodeById.set(payload.nodeId, new AXNode(realm, payload)); } for (const node of nodeById.values()) { for (const childId of node.payload.childIds || []) { diff --git a/packages/puppeteer-core/src/cdp/Frame.ts b/packages/puppeteer-core/src/cdp/Frame.ts index 65655944112..5a145241b53 100644 --- a/packages/puppeteer-core/src/cdp/Frame.ts +++ b/packages/puppeteer-core/src/cdp/Frame.ts @@ -15,6 +15,7 @@ import {Deferred} from '../util/Deferred.js'; import {disposeSymbol} from '../util/disposable.js'; import {isErrorLike} from '../util/ErrorLike.js'; +import {Accessibility} from './Accessibility.js'; import type { DeviceRequestPrompt, DeviceRequestPromptManager, @@ -44,6 +45,7 @@ export class CdpFrame extends Frame { override _id: string; override _parentId?: string; + override accessibility: Accessibility; worlds: IsolatedWorldChart; @@ -70,6 +72,8 @@ export class CdpFrame extends Frame { ), }; + this.accessibility = new Accessibility(this.worlds[MAIN_WORLD]); + this.on(FrameEvent.FrameSwappedByActivation, () => { // Emulate loading process for swapped frames. this._onLoadingStarted(); diff --git a/packages/puppeteer-core/src/cdp/Page.ts b/packages/puppeteer-core/src/cdp/Page.ts index 69c5ed4acd0..7de970a67f5 100644 --- a/packages/puppeteer-core/src/cdp/Page.ts +++ b/packages/puppeteer-core/src/cdp/Page.ts @@ -56,7 +56,6 @@ import {Deferred} from '../util/Deferred.js'; import {AsyncDisposableStack} from '../util/disposable.js'; import {isErrorLike} from '../util/ErrorLike.js'; -import {Accessibility} from './Accessibility.js'; import {Binding} from './Binding.js'; import {CdpCDPSession} from './CDPSession.js'; import {isTargetClosedError} from './Connection.js'; @@ -128,7 +127,6 @@ export class CdpPage extends Page { #keyboard: CdpKeyboard; #mouse: CdpMouse; #touchscreen: CdpTouchscreen; - #accessibility: Accessibility; #frameManager: FrameManager; #emulationManager: EmulationManager; #tracing: Tracing; @@ -237,7 +235,6 @@ export class CdpPage extends Page { this.#keyboard = new CdpKeyboard(client); this.#mouse = new CdpMouse(client, this.#keyboard); this.#touchscreen = new CdpTouchscreen(client, this.#keyboard); - this.#accessibility = new Accessibility(client); this.#frameManager = new FrameManager(client, this, this._timeoutSettings); this.#emulationManager = new EmulationManager(client); this.#tracing = new Tracing(client); @@ -315,7 +312,6 @@ export class CdpPage extends Page { this.#keyboard.updateClient(newSession); this.#mouse.updateClient(newSession); this.#touchscreen.updateClient(newSession); - this.#accessibility.updateClient(newSession); this.#emulationManager.updateClient(newSession); this.#tracing.updateClient(newSession); this.#coverage.updateClient(newSession); @@ -523,10 +519,6 @@ export class CdpPage extends Page { return this.#tracing; } - override get accessibility(): Accessibility { - return this.#accessibility; - } - override frames(): Frame[] { return this.#frameManager.frames(); } diff --git a/test/src/accessibility.spec.ts b/test/src/accessibility.spec.ts index 8c77df56a59..a2547a5023d 100644 --- a/test/src/accessibility.spec.ts +++ b/test/src/accessibility.spec.ts @@ -294,7 +294,7 @@ describe('Accessibility', function () { }, ], }; - expect(await page.accessibility.snapshot()).toEqual(golden); + expect(await page.accessibility.snapshot()).toMatchObject(golden); }); it('rich text editable fields should have children', async () => { const {page, isFirefox} = await getTestState(); @@ -386,7 +386,7 @@ describe('Accessibility', function () { const snapshot = await page.accessibility.snapshot(); assert(snapshot); assert(snapshot.children); - expect(snapshot.children[0]).toEqual({ + expect(snapshot.children[0]).toMatchObject({ role: 'textbox', name: '', value: 'Edit this image:', @@ -416,7 +416,7 @@ describe('Accessibility', function () { const snapshot = await page.accessibility.snapshot(); assert(snapshot); assert(snapshot.children); - expect(snapshot.children[0]).toEqual(golden); + expect(snapshot.children[0]).toMatchObject(golden); }); it('checkbox with and tabIndex and label should not have children', async () => { const {page, isFirefox} = await getTestState(); @@ -440,7 +440,7 @@ describe('Accessibility', function () { const snapshot = await page.accessibility.snapshot(); assert(snapshot); assert(snapshot.children); - expect(snapshot.children[0]).toEqual(golden); + expect(snapshot.children[0]).toMatchObject(golden); }); it('checkbox without label should not have children', async () => { const {page, isFirefox} = await getTestState(); @@ -464,7 +464,7 @@ describe('Accessibility', function () { const snapshot = await page.accessibility.snapshot(); assert(snapshot); assert(snapshot.children); - expect(snapshot.children[0]).toEqual(golden); + expect(snapshot.children[0]).toMatchObject(golden); }); describe('root option', function () { @@ -474,10 +474,12 @@ describe('Accessibility', function () { await page.setContent(``); using button = (await page.$('button'))!; - expect(await page.accessibility.snapshot({root: button})).toEqual({ - role: 'button', - name: 'My Button', - }); + expect(await page.accessibility.snapshot({root: button})).toMatchObject( + { + role: 'button', + name: 'My Button', + } + ); }); it('should work an input', async () => { const {page} = await getTestState(); @@ -485,7 +487,7 @@ describe('Accessibility', function () { await page.setContent(``); using input = (await page.$('input'))!; - expect(await page.accessibility.snapshot({root: input})).toEqual({ + expect(await page.accessibility.snapshot({root: input})).toMatchObject({ role: 'textbox', name: 'My Input', value: 'My Value', @@ -503,7 +505,7 @@ describe('Accessibility', function () { `); using menu = (await page.$('div[role="menu"]'))!; - expect(await page.accessibility.snapshot({root: menu})).toEqual({ + expect(await page.accessibility.snapshot({root: menu})).toMatchObject({ role: 'menu', name: 'My Menu', children: [ @@ -548,6 +550,28 @@ describe('Accessibility', function () { }); }); }); + + describe('elementHandle()', () => { + it('should get an ElementHandle from a snapshot item', async () => { + const {page} = await getTestState(); + + await page.setContent(``); + + using button = (await page.$('button'))!; + const snapshot = await page.accessibility.snapshot({root: button}); + expect(snapshot).toMatchObject({ + role: 'button', + name: 'My Button', + }); + + using buttonHandle = await snapshot!.elementHandle(); + expect( + await buttonHandle?.evaluate(button => { + return button.innerHTML; + }) + ).toEqual('My Button'); + }); + }); }); function findFocusedNode(