chore: implement ElementHandle.prototype.clickablePoint (#10775)

packages/puppeteer-core/src/api/ElementHandle.ts

@@ -630,9 +630,21 @@ export abstract class ElementHandle<
   /**
    * Returns the middle point within an element unless a specific offset is provided.
    */
-  async clickablePoint(offset?: Offset): Promise<Point>;
+  async clickablePoint(offset?: Offset): Promise<Point> {
-  async clickablePoint(): Promise<Point> {
+    const box = await this.#clickableBox();
-    throw new Error('Not implemented');
+    if (!box) {
+      throw new Error('Node is either not clickable or not an Element');
+    }
+    if (offset !== undefined) {
+      return {
+        x: box.x + offset.x,
+        y: box.y + offset.y,
+      };
+    }
+    return {
+      x: box.x + box.width / 2,
+      y: box.y + box.height / 2,
+    };
   }
 
   /**
@@ -861,43 +873,21 @@ export abstract class ElementHandle<
     options?: Readonly<KeyPressOptions>
   ): Promise<void>;
 
-  /**
+  async #clickableBox(): Promise<BoundingBox | null> {
-   * This method returns the bounding box of the element (relative to the main frame),
-   * or `null` if the element is not visible.
-   */
-  async boundingBox(): Promise<BoundingBox | null> {
     const adoptedThis = await this.frame.isolatedRealm().adoptHandle(this);
-    const box = await adoptedThis.evaluate(element => {
+    const rects = await adoptedThis.evaluate(element => {
       if (!(element instanceof Element)) {
         return null;
       }
-      // Element is not visible.
+      return [...element.getClientRects()].map(rect => {
-      if (element.getClientRects().length === 0) {
+        return rect.toJSON();
-        return null;
+      }) as DOMRect[];
-      }
-      const rect = element.getBoundingClientRect();
-      return {
-        x: rect.left,
-        y: rect.top,
-        width: rect.width,
-        height: rect.height,
-      };
     });
     void adoptedThis.dispose().catch(debugError);
-    if (!box) {
+    if (!rects?.length) {
       return null;
     }
-    const offset = await this.#getTopLeftCornerOfFrame();
+    await this.#intersectBoundingBoxesWithFrame(rects);
-    if (!offset) {
-      return null;
-    }
-    box.x += offset.x;
-    box.y += offset.y;
-    return box;
-  }
-
-  async #getTopLeftCornerOfFrame() {
-    const point = {x: 0, y: 0};
     let frame: Frame | null | undefined = this.frame;
     let element: HandleFor<HTMLIFrameElement> | null | undefined;
     while ((element = await frame?.frameElement())) {
@@ -924,14 +914,77 @@ export abstract class ElementHandle<
         if (!parentBox) {
           return null;
         }
-        point.x += parentBox.left;
+        for (const box of rects) {
-        point.y += parentBox.top;
+          box.x += parentBox.left;
+          box.y += parentBox.top;
+        }
+        await element.#intersectBoundingBoxesWithFrame(rects);
         frame = frame?.parentFrame();
       } finally {
         void element.dispose().catch(debugError);
       }
     }
-    return point;
+    const rect = rects.find(box => {
+      return box.width >= 1 && box.height >= 1;
+    });
+    if (!rect) {
+      return null;
+    }
+    return {
+      x: rect.x,
+      y: rect.y,
+      height: rect.height,
+      width: rect.width,
+    };
+  }
+
+  async #intersectBoundingBoxesWithFrame(boxes: BoundingBox[]) {
+    const {documentWidth, documentHeight} = await this.frame
+      .isolatedRealm()
+      .evaluate(() => {
+        return {
+          documentWidth: document.documentElement.clientWidth,
+          documentHeight: document.documentElement.clientHeight,
+        };
+      });
+    for (const box of boxes) {
+      intersectBoundingBox(box, documentWidth, documentHeight);
+    }
+  }
+
+  /**
+   * This method returns the bounding box of the element (relative to the main frame),
+   * or `null` if the element is not visible.
+   */
+  async boundingBox(): Promise<BoundingBox | null> {
+    const adoptedThis = await this.frame.isolatedRealm().adoptHandle(this);
+    const box = await adoptedThis.evaluate(element => {
+      if (!(element instanceof Element)) {
+        return null;
+      }
+      // Element is not visible.
+      if (element.getClientRects().length === 0) {
+        return null;
+      }
+      const rect = element.getBoundingClientRect();
+      return rect.toJSON() as DOMRect;
+    });
+    void adoptedThis.dispose().catch(debugError);
+    if (!box) {
+      return null;
+    }
+    const offset = await this.#getTopLeftCornerOfFrame();
+    if (!offset) {
+      return null;
+    }
+    box.x += offset.x;
+    box.y += offset.y;
+    return {
+      x: box.x,
+      y: box.y,
+      height: box.height,
+      width: box.width,
+    };
   }
 
   /**
@@ -1038,6 +1091,44 @@ export abstract class ElementHandle<
     return model;
   }
 
+  async #getTopLeftCornerOfFrame() {
+    const point = {x: 0, y: 0};
+    let frame: Frame | null | undefined = this.frame;
+    let element: HandleFor<HTMLIFrameElement> | null | undefined;
+    while ((element = await frame?.frameElement())) {
+      try {
+        element = await element.frame.isolatedRealm().transferHandle(element);
+        const parentBox = await element.evaluate(element => {
+          // Element is not visible.
+          if (element.getClientRects().length === 0) {
+            return null;
+          }
+          const rect = element.getBoundingClientRect();
+          const style = window.getComputedStyle(element);
+          return {
+            left:
+              rect.left +
+              parseInt(style.paddingLeft, 10) +
+              parseInt(style.borderLeftWidth, 10),
+            top:
+              rect.top +
+              parseInt(style.paddingTop, 10) +
+              parseInt(style.borderTopWidth, 10),
+          };
+        });
+        if (!parentBox) {
+          return null;
+        }
+        point.x += parentBox.left;
+        point.y += parentBox.top;
+        frame = frame?.parentFrame();
+      } finally {
+        void element.dispose().catch(debugError);
+      }
+    }
+    return point;
+  }
+
   /**
    * This method scrolls element into view if needed, and then uses
    * {@link Page.(screenshot:3) } to take a screenshot of the element.
@@ -1219,3 +1310,22 @@ export interface AutofillData {
     cvc: string;
   };
 }
+
+function intersectBoundingBox(
+  box: BoundingBox,
+  width: number,
+  height: number
+): void {
+  box.width = Math.max(
+    box.x >= 0
+      ? Math.min(width - box.x, box.width)
+      : Math.min(width, box.width + box.x),
+    0
+  );
+  box.height = Math.max(
+    box.y >= 0
+      ? Math.min(height - box.y, box.height)
+      : Math.min(height, box.height + box.y),
+    0
+  );
+}

packages/puppeteer-core/src/common/ElementHandle.ts

@@ -20,9 +20,7 @@ import {
   AutofillData,
   ClickOptions,
   ElementHandle,
-  Offset,
   Point,
-  Quad,
 } from '../api/ElementHandle.js';
 import {KeyboardTypeOptions, KeyPressOptions} from '../api/Input.js';
 import {Page, ScreenshotOptions} from '../api/Page.js';
@@ -34,23 +32,10 @@ import {Frame} from './Frame.js';
 import {FrameManager} from './FrameManager.js';
 import {WaitForSelectorOptions} from './IsolatedWorld.js';
 import {CDPJSHandle} from './JSHandle.js';
-import {CDPPage} from './Page.js';
 import {NodeFor} from './types.js';
 import {KeyInput} from './USKeyboardLayout.js';
 import {debugError} from './util.js';
 
-const applyOffsetsToQuad = (
-  quad: Point[],
-  offsetX: number,
-  offsetY: number
-) => {
-  assert(quad.length === 4);
-  return quad.map(part => {
-    return {x: part.x + offsetX, y: part.y + offsetY};
-    // SAFETY: We know this is a quad from the length check.
-  }) as Quad;
-};
-
 /**
  * The CDPElementHandle extends ElementHandle now to keep compatibility
  * with `instanceof` because of that we need to have methods for
@@ -156,127 +141,6 @@ export class CDPElementHandle<
     }
   }
 
-  async #getOOPIFOffsets(
-    frame: Frame
-  ): Promise<{offsetX: number; offsetY: number}> {
-    let offsetX = 0;
-    let offsetY = 0;
-    let currentFrame: Frame | null = frame;
-    while (currentFrame && currentFrame.parentFrame()) {
-      const parent = currentFrame.parentFrame();
-      if (!currentFrame.isOOPFrame() || !parent) {
-        currentFrame = parent;
-        continue;
-      }
-      const {backendNodeId} = await parent._client().send('DOM.getFrameOwner', {
-        frameId: currentFrame._id,
-      });
-      const result = await parent._client().send('DOM.getBoxModel', {
-        backendNodeId: backendNodeId,
-      });
-      if (!result) {
-        break;
-      }
-      const contentBoxQuad = result.model.content;
-      const topLeftCorner = this.#fromProtocolQuad(contentBoxQuad)[0];
-      offsetX += topLeftCorner!.x;
-      offsetY += topLeftCorner!.y;
-      currentFrame = parent;
-    }
-    return {offsetX, offsetY};
-  }
-
-  override async clickablePoint(offset?: Offset): Promise<Point> {
-    const [result, layoutMetrics] = await Promise.all([
-      this.client
-        .send('DOM.getContentQuads', {
-          objectId: this.id,
-        })
-        .catch(debugError),
-      (this.#page as CDPPage)._client().send('Page.getLayoutMetrics'),
-    ]);
-    if (!result || !result.quads.length) {
-      throw new Error('Node is either not clickable or not an HTMLElement');
-    }
-    // Filter out quads that have too small area to click into.
-    // Fallback to `layoutViewport` in case of using Firefox.
-    const {clientWidth, clientHeight} =
-      layoutMetrics.cssLayoutViewport || layoutMetrics.layoutViewport;
-    const {offsetX, offsetY} = await this.#getOOPIFOffsets(this.#frame);
-    const quads = result.quads
-      .map(quad => {
-        return this.#fromProtocolQuad(quad);
-      })
-      .map(quad => {
-        return applyOffsetsToQuad(quad, offsetX, offsetY);
-      })
-      .map(quad => {
-        return this.#intersectQuadWithViewport(quad, clientWidth, clientHeight);
-      })
-      .filter(quad => {
-        return computeQuadArea(quad) > 1;
-      });
-    if (!quads.length) {
-      throw new Error('Node is either not clickable or not an HTMLElement');
-    }
-    const quad = quads[0]!;
-    if (offset) {
-      // Return the point of the first quad identified by offset.
-      let minX = Number.MAX_SAFE_INTEGER;
-      let minY = Number.MAX_SAFE_INTEGER;
-      for (const point of quad) {
-        if (point.x < minX) {
-          minX = point.x;
-        }
-        if (point.y < minY) {
-          minY = point.y;
-        }
-      }
-      if (
-        minX !== Number.MAX_SAFE_INTEGER &&
-        minY !== Number.MAX_SAFE_INTEGER
-      ) {
-        return {
-          x: minX + offset.x,
-          y: minY + offset.y,
-        };
-      }
-    }
-    // Return the middle point of the first quad.
-    let x = 0;
-    let y = 0;
-    for (const point of quad) {
-      x += point.x;
-      y += point.y;
-    }
-    return {
-      x: x / 4,
-      y: y / 4,
-    };
-  }
-
-  #fromProtocolQuad(quad: number[]): Point[] {
-    return [
-      {x: quad[0]!, y: quad[1]!},
-      {x: quad[2]!, y: quad[3]!},
-      {x: quad[4]!, y: quad[5]!},
-      {x: quad[6]!, y: quad[7]!},
-    ];
-  }
-
-  #intersectQuadWithViewport(
-    quad: Point[],
-    width: number,
-    height: number
-  ): Point[] {
-    return quad.map(point => {
-      return {
-        x: Math.min(Math.max(point.x, 0), width),
-        y: Math.min(Math.max(point.y, 0), height),
-      };
-    });
-  }
-
   /**
    * This method scrolls element into view if needed, and then
    * uses {@link Page.mouse} to hover over the center of the element.
@@ -532,16 +396,3 @@ export class CDPElementHandle<
     assert(this.executionContext()._world);
   }
 }
-
-function computeQuadArea(quad: Point[]): number {
-  /* Compute sum of all directed areas of adjacent triangles
-     https://en.wikipedia.org/wiki/Polygon#Simple_polygons
-   */
-  let area = 0;
-  for (let i = 0; i < quad.length; ++i) {
-    const p1 = quad[i]!;
-    const p2 = quad[(i + 1) % quad.length]!;
-    area += (p1.x * p2.y - p2.x * p1.y) / 2;
-  }
-  return Math.abs(area);
-}

packages/puppeteer-core/src/common/IsolatedWorld.ts

@@ -500,10 +500,16 @@ export class IsolatedWorld implements Realm {
 
   async adoptHandle<T extends JSHandle<Node>>(handle: T): Promise<T> {
     const context = await this.executionContext();
-    assert(
+    if (
-      (handle as unknown as CDPJSHandle<Node>).executionContext() !== context,
+      (handle as unknown as CDPJSHandle<Node>).executionContext() === context
-      'Cannot adopt handle that already belongs to this execution context'
+    ) {
-    );
+      // If the context has already adopted this handle, clone it so downstream
+      // disposal doesn't become an issue.
+      return (await handle.evaluateHandle(value => {
+        return value;
+        // SAFETY: We know the
+      })) as unknown as T;
+    }
     const nodeInfo = await this.#client.send('DOM.describeNode', {
       objectId: handle.id,
     });

test/TestExpectations.json

@@ -623,6 +623,12 @@
     "parameters": ["webDriverBiDi"],
     "expectations": ["PASS"]
   },
+  {
+    "testIdPattern": "[elementhandle.spec] ElementHandle specs ElementHandle.boundingBox should work",
+    "platforms": ["darwin", "linux", "win32"],
+    "parameters": ["firefox"],
+    "expectations": ["FAIL", "PASS"]
+  },
   {
     "testIdPattern": "[elementhandle.spec] ElementHandle specs ElementHandle.boundingBox should work with SVG nodes",
     "platforms": ["darwin", "linux", "win32"],
@@ -665,6 +671,12 @@
     "parameters": ["webDriverBiDi"],
     "expectations": ["PASS"]
   },
+  {
+    "testIdPattern": "[elementhandle.spec] ElementHandle specs ElementHandle.clickablePoint should work",
+    "platforms": ["darwin", "linux", "win32"],
+    "parameters": ["webDriverBiDi"],
+    "expectations": ["PASS"]
+  },
   {
     "testIdPattern": "[elementhandle.spec] ElementHandle specs ElementHandle.contentFrame should work",
     "platforms": ["darwin", "linux", "win32"],
@@ -2075,18 +2087,6 @@
     "parameters": ["cdp", "firefox"],
     "expectations": ["FAIL"]
   },
-  {
-    "testIdPattern": "[elementhandle.spec] ElementHandle specs ElementHandle.boundingBox should work",
-    "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["firefox", "webDriverBiDi"],
-    "expectations": ["FAIL"]
-  },
-  {
-    "testIdPattern": "[elementhandle.spec] ElementHandle specs ElementHandle.boundingBox should work",
-    "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["cdp", "firefox"],
-    "expectations": ["FAIL", "PASS"]
-  },
   {
     "testIdPattern": "[elementhandle.spec] ElementHandle specs ElementHandle.boxModel should work",
     "platforms": ["darwin", "linux", "win32"],
@@ -2111,6 +2111,12 @@
     "parameters": ["chrome", "webDriverBiDi"],
     "expectations": ["PASS"]
   },
+  {
+    "testIdPattern": "[elementhandle.spec] ElementHandle specs ElementHandle.clickablePoint should work for iframes",
+    "platforms": ["darwin", "linux", "win32"],
+    "parameters": ["chrome", "webDriverBiDi"],
+    "expectations": ["PASS"]
+  },
   {
     "testIdPattern": "[elementhandle.spec] ElementHandle specs ElementHandle.contentFrame should work",
     "platforms": ["darwin", "linux", "win32"],
@@ -2141,6 +2147,12 @@
     "parameters": ["cdp", "firefox"],
     "expectations": ["FAIL"]
   },
+  {
+    "testIdPattern": "[elementhandle.spec] ElementHandle specs ElementHandle.clickablePoint should not work if the click box is not visible",
+    "platforms": ["darwin", "linux", "win32"],
+    "parameters": ["webDriverBiDi"],
+    "expectations": ["PASS"]
+  },
   {
     "testIdPattern": "[emulation.spec] Emulation Page.emulateMediaFeatures should throw in case of bad argument",
     "platforms": ["darwin", "linux", "win32"],

test/src/elementhandle.spec.ts

@@ -294,7 +294,7 @@ describe('ElementHandle specs', function () {
         return error_;
       });
       expect(error.message).atLeastOneToContain([
-        'Node is either not clickable or not an HTMLElement',
+        'Node is either not clickable or not an Element',
         'no such element',
       ]);
     });
@@ -310,7 +310,7 @@ describe('ElementHandle specs', function () {
         return error_;
       });
       expect(error.message).atLeastOneToContain([
-        'Node is either not clickable or not an HTMLElement',
+        'Node is either not clickable or not an Element',
         'no such element',
       ]);
     });
@@ -323,7 +323,7 @@ describe('ElementHandle specs', function () {
         return error_;
       });
       expect(error.message).atLeastOneToContain([
-        'Node is either not clickable or not an HTMLElement',
+        'Node is either not clickable or not an Element',
         'no such node',
       ]);
     });
@@ -361,6 +361,58 @@ describe('ElementHandle specs', function () {
       });
     });
 
+    it('should not work if the click box is not visible', async () => {
+      const {page} = await getTestState();
+
+      await page.setContent(
+        '<button style="width: 10px; height: 10px; position: absolute; left: -20px"></button>'
+      );
+      const handle = await page.locator('button').waitHandle();
+      await expect(handle.clickablePoint()).rejects.toBeInstanceOf(Error);
+
+      await page.setContent(
+        '<button style="width: 10px; height: 10px; position: absolute; right: -20px"></button>'
+      );
+      const handle2 = await page.locator('button').waitHandle();
+      await expect(handle2.clickablePoint()).rejects.toBeInstanceOf(Error);
+
+      await page.setContent(
+        '<button style="width: 10px; height: 10px; position: absolute; top: -20px"></button>'
+      );
+      const handle3 = await page.locator('button').waitHandle();
+      await expect(handle3.clickablePoint()).rejects.toBeInstanceOf(Error);
+
+      await page.setContent(
+        '<button style="width: 10px; height: 10px; position: absolute; bottom: -20px"></button>'
+      );
+      const handle4 = await page.locator('button').waitHandle();
+      await expect(handle4.clickablePoint()).rejects.toBeInstanceOf(Error);
+    });
+
+    it('should not work if the click box is not visible due to the iframe', async () => {
+      const {page} = await getTestState();
+
+      await page.setContent(
+        `<iframe name='frame' style='position: absolute; left: -100px' srcdoc="<button style='width: 10px; height: 10px;'></button>"></iframe>`
+      );
+      const frame = await page.waitForFrame(frame => {
+        return frame.name() === 'frame';
+      });
+
+      const handle = await frame.locator('button').waitHandle();
+      await expect(handle.clickablePoint()).rejects.toBeInstanceOf(Error);
+
+      await page.setContent(
+        `<iframe name='frame2' style='position: absolute; top: -100px' srcdoc="<button style='width: 10px; height: 10px;'></button>"></iframe>`
+      );
+      const frame2 = await page.waitForFrame(frame => {
+        return frame.name() === 'frame2';
+      });
+
+      const handle2 = await frame2.locator('button').waitHandle();
+      await expect(handle2.clickablePoint()).rejects.toBeInstanceOf(Error);
+    });
+
     it('should work for iframes', async () => {
       const {page} = await getTestState();
       await page.evaluate(() => {