From 34db39e4474efee9d4579743026c3d6b6c8e494b Mon Sep 17 00:00:00 2001 From: jrandolf <101637635+jrandolf@users.noreply.github.com> Date: Mon, 17 Apr 2023 10:56:51 +0200 Subject: [PATCH] fix: improve mouse actions (#10021) --- docs/api/index.md | 3 + docs/api/puppeteer.mouse.click.md | 18 +- docs/api/puppeteer.mouse.down.md | 8 +- docs/api/puppeteer.mouse.md | 6 +- docs/api/puppeteer.mouse.move.md | 20 +- docs/api/puppeteer.mouse.up.md | 8 +- docs/api/puppeteer.mousebutton.md | 12 +- docs/api/puppeteer.mouseclickoptions.md | 19 ++ docs/api/puppeteer.mousemoveoptions.md | 17 ++ docs/api/puppeteer.mouseoptions.md | 8 +- packages/puppeteer-core/src/common/Input.ts | 302 +++++++++++++++----- test/src/mouse.spec.ts | 92 ++++++ 12 files changed, 402 insertions(+), 111 deletions(-) create mode 100644 docs/api/puppeteer.mouseclickoptions.md create mode 100644 docs/api/puppeteer.mousemoveoptions.md diff --git a/docs/api/index.md b/docs/api/index.md index a2f5d480..e90971ab 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -97,6 +97,8 @@ sidebar_label: API | [LaunchOptions](./puppeteer.launchoptions.md) | Generic launch options that can be passed when launching any browser. | | [MediaFeature](./puppeteer.mediafeature.md) | | | [Metrics](./puppeteer.metrics.md) | | +| [MouseClickOptions](./puppeteer.mouseclickoptions.md) | | +| [MouseMoveOptions](./puppeteer.mousemoveoptions.md) | | | [MouseOptions](./puppeteer.mouseoptions.md) | | | [MouseWheelOptions](./puppeteer.mousewheeloptions.md) | | | [NetworkConditions](./puppeteer.networkconditions.md) | | @@ -135,6 +137,7 @@ sidebar_label: API | [executablePath](./puppeteer.executablepath.md) | | | [KnownDevices](./puppeteer.knowndevices.md) | A list of devices to be used with [Page.emulate()](./puppeteer.page.emulate.md). | | [launch](./puppeteer.launch.md) | | +| [MouseButton](./puppeteer.mousebutton.md) | Enum of valid mouse buttons. | | [networkConditions](./puppeteer.networkconditions.md) | | | [PredefinedNetworkConditions](./puppeteer.predefinednetworkconditions.md) | A list of network conditions to be used with [Page.emulateNetworkConditions()](./puppeteer.page.emulatenetworkconditions.md). | | [puppeteer](./puppeteer.puppeteer.md) | | diff --git a/docs/api/puppeteer.mouse.click.md b/docs/api/puppeteer.mouse.click.md index c64923bc..0ad05c0d 100644 --- a/docs/api/puppeteer.mouse.click.md +++ b/docs/api/puppeteer.mouse.click.md @@ -10,23 +10,17 @@ Shortcut for `mouse.move`, `mouse.down` and `mouse.up`. ```typescript class Mouse { - click( - x: number, - y: number, - options?: MouseOptions & { - delay?: number; - } - ): Promise; + click(x: number, y: number, options?: MouseClickOptions): Promise; } ``` ## Parameters -| Parameter | Type | Description | -| --------- | --------------------------------------------------------------------- | ------------------------------------------------ | -| x | number | Horizontal position of the mouse. | -| y | number | Vertical position of the mouse. | -| options | [MouseOptions](./puppeteer.mouseoptions.md) & { delay?: number; } | _(Optional)_ Optional MouseOptions. | +| Parameter | Type | Description | +| --------- | ----------------------------------------------------- | ------------------------------------------- | +| x | number | Horizontal position of the mouse. | +| y | number | Vertical position of the mouse. | +| options | [MouseClickOptions](./puppeteer.mouseclickoptions.md) | _(Optional)_ Options to configure behavior. | **Returns:** diff --git a/docs/api/puppeteer.mouse.down.md b/docs/api/puppeteer.mouse.down.md index 7274efe4..5116b9ce 100644 --- a/docs/api/puppeteer.mouse.down.md +++ b/docs/api/puppeteer.mouse.down.md @@ -4,7 +4,7 @@ sidebar_label: Mouse.down # Mouse.down() method -Dispatches a `mousedown` event. +Presses the mouse. #### Signature: @@ -16,9 +16,9 @@ class Mouse { ## Parameters -| Parameter | Type | Description | -| --------- | ------------------------------------------- | ------------------------------------------------ | -| options | [MouseOptions](./puppeteer.mouseoptions.md) | _(Optional)_ Optional MouseOptions. | +| Parameter | Type | Description | +| --------- | ------------------------------------------- | ------------------------------------------- | +| options | [MouseOptions](./puppeteer.mouseoptions.md) | _(Optional)_ Options to configure behavior. | **Returns:** diff --git a/docs/api/puppeteer.mouse.md b/docs/api/puppeteer.mouse.md index 68de341f..681c3243 100644 --- a/docs/api/puppeteer.mouse.md +++ b/docs/api/puppeteer.mouse.md @@ -80,12 +80,12 @@ await browser | Method | Modifiers | Description | | ----------------------------------------------------------------------- | --------- | ---------------------------------------------------------------------------------------- | | [click(x, y, options)](./puppeteer.mouse.click.md) | | Shortcut for mouse.move, mouse.down and mouse.up. | -| [down(options)](./puppeteer.mouse.down.md) | | Dispatches a mousedown event. | +| [down(options)](./puppeteer.mouse.down.md) | | Presses the mouse. | | [drag(start, target)](./puppeteer.mouse.drag.md) | | Dispatches a drag event. | | [dragAndDrop(start, target, options)](./puppeteer.mouse.draganddrop.md) | | Performs a drag, dragenter, dragover, and drop in sequence. | | [dragEnter(target, data)](./puppeteer.mouse.dragenter.md) | | Dispatches a dragenter event. | | [dragOver(target, data)](./puppeteer.mouse.dragover.md) | | Dispatches a dragover event. | | [drop(target, data)](./puppeteer.mouse.drop.md) | | Performs a dragenter, dragover, and drop in sequence. | -| [move(x, y, options)](./puppeteer.mouse.move.md) | | Dispatches a mousemove event. | -| [up(options)](./puppeteer.mouse.up.md) | | Dispatches a mouseup event. | +| [move(x, y, options)](./puppeteer.mouse.move.md) | | Moves the mouse to the given coordinate. | +| [up(options)](./puppeteer.mouse.up.md) | | Releases the mouse. | | [wheel(options)](./puppeteer.mouse.wheel.md) | | Dispatches a mousewheel event. | diff --git a/docs/api/puppeteer.mouse.move.md b/docs/api/puppeteer.mouse.move.md index 1fe74e81..01ca4109 100644 --- a/docs/api/puppeteer.mouse.move.md +++ b/docs/api/puppeteer.mouse.move.md @@ -4,29 +4,23 @@ sidebar_label: Mouse.move # Mouse.move() method -Dispatches a `mousemove` event. +Moves the mouse to the given coordinate. #### Signature: ```typescript class Mouse { - move( - x: number, - y: number, - options?: { - steps?: number; - } - ): Promise; + move(x: number, y: number, options?: MouseMoveOptions): Promise; } ``` ## Parameters -| Parameter | Type | Description | -| --------- | ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| x | number | Horizontal position of the mouse. | -| y | number | Vertical position of the mouse. | -| options | { steps?: number; } | _(Optional)_ Optional object. If specified, the steps property sends intermediate mousemove events when set to 1 (default). | +| Parameter | Type | Description | +| --------- | --------------------------------------------------- | ------------------------------------------- | +| x | number | Horizontal position of the mouse. | +| y | number | Vertical position of the mouse. | +| options | [MouseMoveOptions](./puppeteer.mousemoveoptions.md) | _(Optional)_ Options to configure behavior. | **Returns:** diff --git a/docs/api/puppeteer.mouse.up.md b/docs/api/puppeteer.mouse.up.md index abe71eda..e460f8a9 100644 --- a/docs/api/puppeteer.mouse.up.md +++ b/docs/api/puppeteer.mouse.up.md @@ -4,7 +4,7 @@ sidebar_label: Mouse.up # Mouse.up() method -Dispatches a `mouseup` event. +Releases the mouse. #### Signature: @@ -16,9 +16,9 @@ class Mouse { ## Parameters -| Parameter | Type | Description | -| --------- | ------------------------------------------- | ------------------------------------------------ | -| options | [MouseOptions](./puppeteer.mouseoptions.md) | _(Optional)_ Optional MouseOptions. | +| Parameter | Type | Description | +| --------- | ------------------------------------------- | ------------------------------------------- | +| options | [MouseOptions](./puppeteer.mouseoptions.md) | _(Optional)_ Options to configure behavior. | **Returns:** diff --git a/docs/api/puppeteer.mousebutton.md b/docs/api/puppeteer.mousebutton.md index 3c531415..4ba4f6fc 100644 --- a/docs/api/puppeteer.mousebutton.md +++ b/docs/api/puppeteer.mousebutton.md @@ -2,10 +2,18 @@ sidebar_label: MouseButton --- -# MouseButton type +# MouseButton variable + +Enum of valid mouse buttons. #### Signature: ```typescript -export type MouseButton = 'left' | 'right' | 'middle' | 'back' | 'forward'; +MouseButton: Readonly<{ + Left: 'left'; + Right: 'right'; + Middle: 'middle'; + Back: 'back'; + Forward: 'forward'; +}>; ``` diff --git a/docs/api/puppeteer.mouseclickoptions.md b/docs/api/puppeteer.mouseclickoptions.md new file mode 100644 index 00000000..971110f2 --- /dev/null +++ b/docs/api/puppeteer.mouseclickoptions.md @@ -0,0 +1,19 @@ +--- +sidebar_label: MouseClickOptions +--- + +# MouseClickOptions interface + +#### Signature: + +```typescript +export interface MouseClickOptions extends MouseOptions +``` + +**Extends:** [MouseOptions](./puppeteer.mouseoptions.md) + +## Properties + +| Property | Modifiers | Type | Description | Default | +| -------- | --------------------- | ------ | -------------------------------------------------------------- | ------- | +| delay | optional | number | Time (in ms) to delay the mouse release after the mouse press. | | diff --git a/docs/api/puppeteer.mousemoveoptions.md b/docs/api/puppeteer.mousemoveoptions.md new file mode 100644 index 00000000..618ba72d --- /dev/null +++ b/docs/api/puppeteer.mousemoveoptions.md @@ -0,0 +1,17 @@ +--- +sidebar_label: MouseMoveOptions +--- + +# MouseMoveOptions interface + +#### Signature: + +```typescript +export interface MouseMoveOptions +``` + +## Properties + +| Property | Modifiers | Type | Description | Default | +| -------- | --------------------- | ------ | ------------------------------------------------------------------------------------------ | -------------- | +| steps | optional | number | Determines the number of movements to make from the current mouse position to the new one. | 1 | diff --git a/docs/api/puppeteer.mouseoptions.md b/docs/api/puppeteer.mouseoptions.md index 2e45c17c..764b355b 100644 --- a/docs/api/puppeteer.mouseoptions.md +++ b/docs/api/puppeteer.mouseoptions.md @@ -12,7 +12,7 @@ export interface MouseOptions ## Properties -| Property | Modifiers | Type | Description | Default | -| ---------- | --------------------- | ----------------------------------------- | ----------- | ------- | -| button | optional | [MouseButton](./puppeteer.mousebutton.md) | | | -| clickCount | optional | number | | | +| Property | Modifiers | Type | Description | Default | +| ---------- | --------------------- | ----------------------------------------- | ----------------------------------------- | ------------------- | +| button | optional | [MouseButton](./puppeteer.mousebutton.md) | Determines which button will be pressed. | 'left' | +| clickCount | optional | number | Determines the click count for the mouse. | 1 | diff --git a/packages/puppeteer-core/src/common/Input.ts b/packages/puppeteer-core/src/common/Input.ts index 67e97020..7047c3f8 100644 --- a/packages/puppeteer-core/src/common/Input.ts +++ b/packages/puppeteer-core/src/common/Input.ts @@ -334,14 +334,29 @@ export class Keyboard { /** * @public */ -export type MouseButton = 'left' | 'right' | 'middle' | 'back' | 'forward'; +export interface MouseOptions { + /** + * Determines which button will be pressed. + * + * @defaultValue `'left'` + */ + button?: MouseButton; + /** + * Determines the click count for the mouse. + * + * @defaultValue `1` + */ + clickCount?: number; +} /** * @public */ -export interface MouseOptions { - button?: MouseButton; - clickCount?: number; +export interface MouseClickOptions extends MouseOptions { + /** + * Time (in ms) to delay the mouse release after the mouse press. + */ + delay?: number; } /** @@ -352,6 +367,69 @@ export interface MouseWheelOptions { deltaY?: number; } +/** + * @public + */ +export interface MouseMoveOptions { + /** + * Determines the number of movements to make from the current mouse position + * to the new one. + * + * @defaultValue `1` + */ + steps?: number; +} + +/** + * Enum of valid mouse buttons. + * + * @public + */ +export const MouseButton = Object.freeze({ + Left: 'left', + Right: 'right', + Middle: 'middle', + Back: 'back', + Forward: 'forward', +}); + +/** + * @public + */ +export type MouseButton = (typeof MouseButton)[keyof typeof MouseButton]; + +/** + * This must follow {@link Protocol.Input.DispatchMouseEventRequest.buttons}. + */ +const enum MouseButtonFlag { + None = 0, + Left = 1, + Right = 1 << 1, + Middle = 1 << 2, + Back = 1 << 3, + Forward = 1 << 4, +} + +const getFlag = (button: MouseButton): MouseButtonFlag => { + switch (button) { + case MouseButton.Left: + return MouseButtonFlag.Left; + case MouseButton.Right: + return MouseButtonFlag.Right; + case MouseButton.Middle: + return MouseButtonFlag.Middle; + case MouseButton.Back: + return MouseButtonFlag.Back; + case MouseButton.Forward: + return MouseButtonFlag.Forward; + } +}; + +interface MouseState { + position: Point; + buttons: number; +} + /** * The Mouse class operates in main-frame CSS pixels * relative to the top-left corner of the viewport. @@ -426,9 +504,6 @@ export interface MouseWheelOptions { export class Mouse { #client: CDPSession; #keyboard: Keyboard; - #x = 0; - #y = 0; - #button: MouseButton | 'none' = 'none'; /** * @internal @@ -438,88 +513,176 @@ export class Mouse { this.#keyboard = keyboard; } + #_state: Readonly = { + position: {x: 0, y: 0}, + buttons: MouseButtonFlag.None, + }; + get #state(): MouseState { + return Object.assign({...this.#_state}, ...this.#transactions); + } + + // Transactions can run in parallel, so we store each of thme in this array. + #transactions: Array> = []; + #createTransaction(): { + update: (updates: Partial) => void; + commit: () => void; + rollback: () => void; + } { + const transaction: Partial = {}; + this.#transactions.push(transaction); + const popTransaction = () => { + this.#transactions.splice(this.#transactions.indexOf(transaction), 1); + }; + return { + update: (updates: Partial) => { + Object.assign(transaction, updates); + }, + commit: () => { + this.#_state = {...this.#_state, ...transaction}; + popTransaction(); + }, + rollback: popTransaction, + }; + } + /** - * Dispatches a `mousemove` event. + * This is a shortcut for a typical update, commit/rollback lifecycle based on + * the error of the action. + */ + async #withTransaction( + action: (update: (updates: Partial) => void) => Promise + ): Promise { + const {update, commit, rollback} = this.#createTransaction(); + try { + await action(update); + commit(); + } catch (error) { + rollback(); + throw error; + } + } + + /** + * Moves the mouse to the given coordinate. + * * @param x - Horizontal position of the mouse. * @param y - Vertical position of the mouse. - * @param options - Optional object. If specified, the `steps` property - * sends intermediate `mousemove` events when set to `1` (default). + * @param options - Options to configure behavior. */ async move( x: number, y: number, - options: {steps?: number} = {} + options: MouseMoveOptions = {} ): Promise { const {steps = 1} = options; - const fromX = this.#x, - fromY = this.#y; - this.#x = x; - this.#y = y; + const from = this.#state.position; + const to = {x, y}; for (let i = 1; i <= steps; i++) { - await this.#client.send('Input.dispatchMouseEvent', { - type: 'mouseMoved', - button: this.#button, - x: fromX + (this.#x - fromX) * (i / steps), - y: fromY + (this.#y - fromY) * (i / steps), - modifiers: this.#keyboard._modifiers, + await this.#withTransaction(updateState => { + updateState({ + position: { + x: from.x + (to.x - from.x) * (i / steps), + y: from.y + (to.y - from.y) * (i / steps), + }, + }); + const {buttons, position} = this.#state; + return this.#client.send('Input.dispatchMouseEvent', { + type: 'mouseMoved', + modifiers: this.#keyboard._modifiers, + buttons, + // This should always be 0 (i.e. 'left'). See + // https://w3c.github.io/uievents/#event-type-mousemove + button: MouseButton.Left, + ...position, + }); }); } } + /** + * Presses the mouse. + * + * @param options - Options to configure behavior. + */ + async down(options: MouseOptions = {}): Promise { + const {button = MouseButton.Left, clickCount = 1} = options; + const flag = getFlag(button); + if (!flag) { + throw new Error(`Unsupported mouse button: ${button}`); + } + if (this.#state.buttons & flag) { + throw new Error(`'${button}' is already pressed.`); + } + await this.#withTransaction(updateState => { + updateState({buttons: this.#state.buttons | flag}); + const {buttons, position} = this.#state; + return this.#client.send('Input.dispatchMouseEvent', { + type: 'mousePressed', + modifiers: this.#keyboard._modifiers, + clickCount, + buttons, + button, + ...position, + }); + }); + } + + /** + * Releases the mouse. + * + * @param options - Options to configure behavior. + */ + async up(options: MouseOptions = {}): Promise { + const {button = MouseButton.Left, clickCount = 1} = options; + const flag = getFlag(button); + if (!flag) { + throw new Error(`Unsupported mouse button: ${button}`); + } + if (!(this.#state.buttons & flag)) { + throw new Error(`'${button}' is not pressed.`); + } + await this.#withTransaction(updateState => { + updateState({buttons: this.#state.buttons & ~flag}); + const {buttons, position} = this.#state; + return this.#client.send('Input.dispatchMouseEvent', { + type: 'mouseReleased', + modifiers: this.#keyboard._modifiers, + clickCount, + buttons, + button, + ...position, + }); + }); + } + /** * Shortcut for `mouse.move`, `mouse.down` and `mouse.up`. + * * @param x - Horizontal position of the mouse. * @param y - Vertical position of the mouse. - * @param options - Optional `MouseOptions`. + * @param options - Options to configure behavior. */ async click( x: number, y: number, - options: MouseOptions & {delay?: number} = {} + options: MouseClickOptions = {} ): Promise { - const {delay = null} = options; - await this.move(x, y); - await this.down(options); - if (delay !== null) { - await new Promise(f => { - return setTimeout(f, delay); + const {delay} = options; + const actions: Array> = []; + const {position} = this.#state; + if (position.x !== x || position.y !== y) { + actions.push(this.move(x, y)); + } + actions.push(this.down(options)); + if (typeof delay === 'number') { + await Promise.all(actions); + actions.length = 0; + await new Promise(resolve => { + setTimeout(resolve, delay); }); } - await this.up(options); - } - - /** - * Dispatches a `mousedown` event. - * @param options - Optional `MouseOptions`. - */ - async down(options: MouseOptions = {}): Promise { - const {button = 'left', clickCount = 1} = options; - this.#button = button; - await this.#client.send('Input.dispatchMouseEvent', { - type: 'mousePressed', - button, - x: this.#x, - y: this.#y, - modifiers: this.#keyboard._modifiers, - clickCount, - }); - } - - /** - * Dispatches a `mouseup` event. - * @param options - Optional `MouseOptions`. - */ - async up(options: MouseOptions = {}): Promise { - const {button = 'left', clickCount = 1} = options; - this.#button = 'none'; - await this.#client.send('Input.dispatchMouseEvent', { - type: 'mouseReleased', - button, - x: this.#x, - y: this.#y, - modifiers: this.#keyboard._modifiers, - clickCount, - }); + actions.push(this.up(options)); + await Promise.all(actions); } /** @@ -546,14 +709,15 @@ export class Mouse { */ async wheel(options: MouseWheelOptions = {}): Promise { const {deltaX = 0, deltaY = 0} = options; + const {position, buttons} = this.#state; await this.#client.send('Input.dispatchMouseEvent', { type: 'mouseWheel', - x: this.#x, - y: this.#y, - deltaX, - deltaY, - modifiers: this.#keyboard._modifiers, pointerType: 'mouse', + modifiers: this.#keyboard._modifiers, + deltaY, + deltaX, + buttons, + ...position, }); } diff --git a/test/src/mouse.spec.ts b/test/src/mouse.spec.ts index f7597c6a..6d7aec47 100644 --- a/test/src/mouse.spec.ts +++ b/test/src/mouse.spec.ts @@ -259,4 +259,96 @@ describe('Mouse', function () { expect(await page.evaluate('result')).toEqual({x: 30, y: 40}); }); + it('should throw if buttons are pressed incorrectly', async () => { + const {page, server} = getTestState(); + + await page.goto(server.EMPTY_PAGE); + + await page.mouse.down(); + await expect(page.mouse.down()).rejects.toBeInstanceOf(Error); + }); + it('should not throw if clicking in parallel', async () => { + const {page, server} = getTestState(); + + await page.goto(server.EMPTY_PAGE); + interface ClickData { + type: string; + detail: number; + clientX: number; + clientY: number; + isTrusted: boolean; + button: number; + buttons: number; + } + + await page.evaluate(() => { + const clicks: ClickData[] = []; + const mouseEventListener = (event: MouseEvent) => { + clicks.push({ + type: event.type, + detail: event.detail, + clientX: event.clientX, + clientY: event.clientY, + isTrusted: event.isTrusted, + button: event.button, + buttons: event.buttons, + }); + }; + document.addEventListener('mousedown', mouseEventListener); + document.addEventListener('mouseup', mouseEventListener); + document.addEventListener('click', mouseEventListener); + (window as unknown as {clicks: ClickData[]}).clicks = clicks; + }); + + await Promise.all([page.mouse.click(0, 5), page.mouse.click(6, 10)]); + + const data = await page.evaluate(() => { + return (window as unknown as {clicks: ClickData[]}).clicks; + }); + const commonAttrs = { + isTrusted: true, + detail: 1, + clientY: 5, + clientX: 0, + button: 0, + }; + expect(data.splice(0, 3)).toMatchObject({ + 0: { + type: 'mousedown', + buttons: 1, + ...commonAttrs, + }, + 1: { + type: 'mouseup', + buttons: 0, + ...commonAttrs, + }, + 2: { + type: 'click', + buttons: 0, + ...commonAttrs, + }, + }); + Object.assign(commonAttrs, { + clientX: 6, + clientY: 10, + }); + expect(data).toMatchObject({ + 0: { + type: 'mousedown', + buttons: 1, + ...commonAttrs, + }, + 1: { + type: 'mouseup', + buttons: 0, + ...commonAttrs, + }, + 2: { + type: 'click', + buttons: 0, + ...commonAttrs, + }, + }); + }); });