feat: allow creating ElementHandles from the accessibility tree snapshot (#12233)

docs/api/puppeteer.serializedaxnode.elementhandle.md

@@ -0,0 +1,21 @@
+---
+sidebar_label: SerializedAXNode.elementHandle
+---
+
+# SerializedAXNode.elementHandle() method
+
+Get an ElementHandle for this AXNode if available.
+
+If the underlying DOM element has been disposed, the method might return an error.
+
+#### Signature:
+
+```typescript
+interface SerializedAXNode {
+  elementHandle(): Promise<ElementHandle | null>;
+}
+```
+
+**Returns:**
+
+Promise&lt;[ElementHandle](./puppeteer.elementhandle.md) \| null&gt;

docs/api/puppeteer.serializedaxnode.md

@@ -502,3 +502,27 @@ A description of the current value.
 
 </td></tr>
 </tbody></table>
+
+## Methods
+
+<table><thead><tr><th>
+
+Method
+
+</th><th>
+
+Description
+
+</th></tr></thead>
+<tbody><tr><td>
+
+<span id="elementhandle">[elementHandle()](./puppeteer.serializedaxnode.elementhandle.md)</span>
+
+</td><td>
+
+Get an ElementHandle for this AXNode if available.
+
+If the underlying DOM element has been disposed, the method might return an error.
+
+</td></tr>
+</tbody></table>

packages/puppeteer-core/src/api/Frame.ts

@@ -14,6 +14,7 @@ import type {
   WaitForSelectorOptions,
   WaitTimeoutOptions,
 } from '../api/Page.js';
+import type {Accessibility} from '../cdp/Accessibility.js';
 import type {DeviceRequestPrompt} from '../cdp/DeviceRequestPrompt.js';
 import type {PuppeteerLifeCycleEvent} from '../cdp/LifecycleWatcher.js';
 import {EventEmitter, type EventType} from '../common/EventEmitter.js';
@@ -379,6 +380,11 @@ export abstract class Frame extends EventEmitter<FrameEvents> {
    */
   abstract get client(): CDPSession;
 
+  /**
+   * @internal
+   */
+  abstract get accessibility(): Accessibility;
+
   /**
    * @internal
    */

packages/puppeteer-core/src/api/Page.ts

@@ -843,7 +843,9 @@ export abstract class Page extends EventEmitter<PageEvents> {
   /**
    * {@inheritDoc Accessibility}
    */
-  abstract get accessibility(): Accessibility;
+  get accessibility(): Accessibility {
+    return this.mainFrame().accessibility;
+  }
 
   /**
    * An array of all frames attached to the page.

packages/puppeteer-core/src/bidi/Frame.ts

@@ -27,6 +27,7 @@ import {
   type WaitForOptions,
 } from '../api/Frame.js';
 import {PageEvent} from '../api/Page.js';
+import {Accessibility} from '../cdp/Accessibility.js';
 import {
   ConsoleMessage,
   type ConsoleMessageLocation,
@@ -71,6 +72,7 @@ export class BidiFrame extends Frame {
 
   override readonly _id: string;
   override readonly client: BidiCdpSession;
+  override readonly accessibility: Accessibility;
 
   private constructor(
     parent: BidiPage | BidiFrame,
@@ -91,6 +93,7 @@ export class BidiFrame extends Frame {
         this
       ),
     };
+    this.accessibility = new Accessibility(this.realms.default);
   }
 
   #initialize(): void {

packages/puppeteer-core/src/bidi/Page.ts

@@ -24,7 +24,6 @@ import {
   type NewDocumentScriptEvaluation,
   type ScreenshotOptions,
 } from '../api/Page.js';
-import {Accessibility} from '../cdp/Accessibility.js';
 import {Coverage} from '../cdp/Coverage.js';
 import {EmulationManager} from '../cdp/EmulationManager.js';
 import {Tracing} from '../cdp/Tracing.js';
@@ -86,7 +85,6 @@ export class BidiPage extends Page {
   readonly keyboard: BidiKeyboard;
   readonly mouse: BidiMouse;
   readonly touchscreen: BidiTouchscreen;
-  readonly accessibility: Accessibility;
   readonly tracing: Tracing;
   readonly coverage: Coverage;
   readonly #cdpEmulationManager: EmulationManager;
@@ -104,7 +102,6 @@ export class BidiPage extends Page {
     this.#frame = BidiFrame.from(this, browsingContext);
 
     this.#cdpEmulationManager = new EmulationManager(this.#frame.client);
-    this.accessibility = new Accessibility(this.#frame.client);
     this.tracing = new Tracing(this.#frame.client);
     this.coverage = new Coverage(this.#frame.client);
     this.keyboard = new BidiKeyboard(this);

packages/puppeteer-core/src/cdp/Accessibility.ts

@@ -6,8 +6,8 @@
 
 import type {Protocol} from 'devtools-protocol';
 
-import type {CDPSession} from '../api/CDPSession.js';
 import type {ElementHandle} from '../api/ElementHandle.js';
+import type {Realm} from '../api/Realm.js';
 
 /**
  * Represents a Node and the properties of it that are relevant to Accessibility.
@@ -80,6 +80,14 @@ export interface SerializedAXNode {
    * Children of this node, if there are any.
    */
   children?: SerializedAXNode[];
+
+  /**
+   * Get an ElementHandle for this AXNode if available.
+   *
+   * If the underlying DOM element has been disposed, the method might return an
+   * error.
+   */
+  elementHandle(): Promise<ElementHandle | null>;
 }
 
 /**
@@ -121,20 +129,13 @@ export interface SnapshotOptions {
  * @public
  */
 export class Accessibility {
-  #client: CDPSession;
+  #realm: Realm;
 
   /**
    * @internal
    */
-  constructor(client: CDPSession) {
-    this.#client = client;
-  }
-
-  /**
-   * @internal
-   */
-  updateClient(client: CDPSession): void {
-    this.#client = client;
+  constructor(realm: Realm) {
+    this.#realm = realm;
   }
 
   /**
@@ -180,15 +181,20 @@ export class Accessibility {
     options: SnapshotOptions = {}
   ): Promise<SerializedAXNode | null> {
     const {interestingOnly = true, root = null} = options;
-    const {nodes} = await this.#client.send('Accessibility.getFullAXTree');
+    const {nodes} = await this.#realm.environment.client.send(
+      'Accessibility.getFullAXTree'
+    );
     let backendNodeId: number | undefined;
     if (root) {
-      const {node} = await this.#client.send('DOM.describeNode', {
+      const {node} = await this.#realm.environment.client.send(
+        'DOM.describeNode',
+        {
           objectId: root.id,
-      });
+        }
+      );
       backendNodeId = node.backendNodeId;
     }
-    const defaultRoot = AXNode.createTree(nodes);
+    const defaultRoot = AXNode.createTree(this.#realm, nodes);
     let needle: AXNode | null = defaultRoot;
     if (backendNodeId) {
       needle = defaultRoot.find(node => {
@@ -260,13 +266,14 @@ class AXNode {
   #role: string;
   #ignored: boolean;
   #cachedHasFocusableChild?: boolean;
+  #realm: Realm;
 
-  constructor(payload: Protocol.Accessibility.AXNode) {
+  constructor(realm: Realm, payload: Protocol.Accessibility.AXNode) {
     this.payload = payload;
     this.#name = this.payload.name ? this.payload.name.value : '';
     this.#role = this.payload.role ? this.payload.role.value : 'Unknown';
     this.#ignored = this.payload.ignored;
-
+    this.#realm = realm;
     for (const property of this.payload.properties || []) {
       if (property.name === 'editable') {
         this.#richlyEditable = property.value.value === 'richtext';
@@ -441,6 +448,14 @@ class AXNode {
 
     const node: SerializedAXNode = {
       role: this.#role,
+      elementHandle: async (): Promise<ElementHandle | null> => {
+        if (!this.payload.backendDOMNodeId) {
+          return null;
+        }
+        return (await this.#realm.adoptBackendNode(
+          this.payload.backendDOMNodeId
+        )) as ElementHandle<Element>;
+      },
     };
 
     type UserStringProperty =
@@ -561,10 +576,13 @@ class AXNode {
     return node;
   }
 
-  public static createTree(payloads: Protocol.Accessibility.AXNode[]): AXNode {
+  public static createTree(
+    realm: Realm,
+    payloads: Protocol.Accessibility.AXNode[]
+  ): AXNode {
     const nodeById = new Map<string, AXNode>();
     for (const payload of payloads) {
-      nodeById.set(payload.nodeId, new AXNode(payload));
+      nodeById.set(payload.nodeId, new AXNode(realm, payload));
     }
     for (const node of nodeById.values()) {
       for (const childId of node.payload.childIds || []) {

packages/puppeteer-core/src/cdp/Frame.ts

@@ -15,6 +15,7 @@ import {Deferred} from '../util/Deferred.js';
 import {disposeSymbol} from '../util/disposable.js';
 import {isErrorLike} from '../util/ErrorLike.js';
 
+import {Accessibility} from './Accessibility.js';
 import type {
   DeviceRequestPrompt,
   DeviceRequestPromptManager,
@@ -44,6 +45,7 @@ export class CdpFrame extends Frame {
 
   override _id: string;
   override _parentId?: string;
+  override accessibility: Accessibility;
 
   worlds: IsolatedWorldChart;
 
@@ -70,6 +72,8 @@ export class CdpFrame extends Frame {
       ),
     };
 
+    this.accessibility = new Accessibility(this.worlds[MAIN_WORLD]);
+
     this.on(FrameEvent.FrameSwappedByActivation, () => {
       // Emulate loading process for swapped frames.
       this._onLoadingStarted();

packages/puppeteer-core/src/cdp/Page.ts

@@ -56,7 +56,6 @@ import {Deferred} from '../util/Deferred.js';
 import {AsyncDisposableStack} from '../util/disposable.js';
 import {isErrorLike} from '../util/ErrorLike.js';
 
-import {Accessibility} from './Accessibility.js';
 import {Binding} from './Binding.js';
 import {CdpCDPSession} from './CDPSession.js';
 import {isTargetClosedError} from './Connection.js';
@@ -128,7 +127,6 @@ export class CdpPage extends Page {
   #keyboard: CdpKeyboard;
   #mouse: CdpMouse;
   #touchscreen: CdpTouchscreen;
-  #accessibility: Accessibility;
   #frameManager: FrameManager;
   #emulationManager: EmulationManager;
   #tracing: Tracing;
@@ -237,7 +235,6 @@ export class CdpPage extends Page {
     this.#keyboard = new CdpKeyboard(client);
     this.#mouse = new CdpMouse(client, this.#keyboard);
     this.#touchscreen = new CdpTouchscreen(client, this.#keyboard);
-    this.#accessibility = new Accessibility(client);
     this.#frameManager = new FrameManager(client, this, this._timeoutSettings);
     this.#emulationManager = new EmulationManager(client);
     this.#tracing = new Tracing(client);
@@ -315,7 +312,6 @@ export class CdpPage extends Page {
     this.#keyboard.updateClient(newSession);
     this.#mouse.updateClient(newSession);
     this.#touchscreen.updateClient(newSession);
-    this.#accessibility.updateClient(newSession);
     this.#emulationManager.updateClient(newSession);
     this.#tracing.updateClient(newSession);
     this.#coverage.updateClient(newSession);
@@ -523,10 +519,6 @@ export class CdpPage extends Page {
     return this.#tracing;
   }
 
-  override get accessibility(): Accessibility {
-    return this.#accessibility;
-  }
-
   override frames(): Frame[] {
     return this.#frameManager.frames();
   }

test/src/accessibility.spec.ts

@@ -294,7 +294,7 @@ describe('Accessibility', function () {
               },
             ],
           };
-      expect(await page.accessibility.snapshot()).toEqual(golden);
+      expect(await page.accessibility.snapshot()).toMatchObject(golden);
     });
     it('rich text editable fields should have children', async () => {
       const {page, isFirefox} = await getTestState();
@@ -386,7 +386,7 @@ describe('Accessibility', function () {
         const snapshot = await page.accessibility.snapshot();
         assert(snapshot);
         assert(snapshot.children);
-        expect(snapshot.children[0]).toEqual({
+        expect(snapshot.children[0]).toMatchObject({
           role: 'textbox',
           name: '',
           value: 'Edit this image:',
@@ -416,7 +416,7 @@ describe('Accessibility', function () {
       const snapshot = await page.accessibility.snapshot();
       assert(snapshot);
       assert(snapshot.children);
-      expect(snapshot.children[0]).toEqual(golden);
+      expect(snapshot.children[0]).toMatchObject(golden);
     });
     it('checkbox with and tabIndex and label should not have children', async () => {
       const {page, isFirefox} = await getTestState();
@@ -440,7 +440,7 @@ describe('Accessibility', function () {
       const snapshot = await page.accessibility.snapshot();
       assert(snapshot);
       assert(snapshot.children);
-      expect(snapshot.children[0]).toEqual(golden);
+      expect(snapshot.children[0]).toMatchObject(golden);
     });
     it('checkbox without label should not have children', async () => {
       const {page, isFirefox} = await getTestState();
@@ -464,7 +464,7 @@ describe('Accessibility', function () {
       const snapshot = await page.accessibility.snapshot();
       assert(snapshot);
       assert(snapshot.children);
-      expect(snapshot.children[0]).toEqual(golden);
+      expect(snapshot.children[0]).toMatchObject(golden);
     });
 
     describe('root option', function () {
@@ -474,10 +474,12 @@ describe('Accessibility', function () {
         await page.setContent(`<button>My Button</button>`);
 
         using button = (await page.$('button'))!;
-        expect(await page.accessibility.snapshot({root: button})).toEqual({
+        expect(await page.accessibility.snapshot({root: button})).toMatchObject(
+          {
             role: 'button',
             name: 'My Button',
-        });
+          }
+        );
       });
       it('should work an input', async () => {
         const {page} = await getTestState();
@@ -485,7 +487,7 @@ describe('Accessibility', function () {
         await page.setContent(`<input title="My Input" value="My Value">`);
 
         using input = (await page.$('input'))!;
-        expect(await page.accessibility.snapshot({root: input})).toEqual({
+        expect(await page.accessibility.snapshot({root: input})).toMatchObject({
           role: 'textbox',
           name: 'My Input',
           value: 'My Value',
@@ -503,7 +505,7 @@ describe('Accessibility', function () {
           `);
 
         using menu = (await page.$('div[role="menu"]'))!;
-        expect(await page.accessibility.snapshot({root: menu})).toEqual({
+        expect(await page.accessibility.snapshot({root: menu})).toMatchObject({
           role: 'menu',
           name: 'My Menu',
           children: [
@@ -548,6 +550,28 @@ describe('Accessibility', function () {
         });
       });
     });
+
+    describe('elementHandle()', () => {
+      it('should get an ElementHandle from a snapshot item', async () => {
+        const {page} = await getTestState();
+
+        await page.setContent(`<button>My Button</button>`);
+
+        using button = (await page.$('button'))!;
+        const snapshot = await page.accessibility.snapshot({root: button});
+        expect(snapshot).toMatchObject({
+          role: 'button',
+          name: 'My Button',
+        });
+
+        using buttonHandle = await snapshot!.elementHandle();
+        expect(
+          await buttonHandle?.evaluate(button => {
+            return button.innerHTML;
+          })
+        ).toEqual('My Button');
+      });
+    });
   });
 
   function findFocusedNode(