From 98ee35655fd203b65df4e157b3cf0215e8636e86 Mon Sep 17 00:00:00 2001 From: JoelEinbinder Date: Fri, 21 Jul 2017 20:29:31 -0700 Subject: [PATCH] Mouse (#101) This patch: - adds Mouse class which holds mouse state and implements mouse primitives, such as moving, button down and button up. - implements high-level mouse api, such as `page.click` and `page.hover`. References #40, References #89 --- docs/api.md | 69 +++++++++++- lib/FrameManager.js | 42 ++++++- lib/Mouse.js | 106 ++++++++++++++++++ lib/Page.js | 63 +++++------ phantom_shim/WebPage.js | 39 +++++++ test/assets/input/button.html | 1 + test/assets/input/mouse-helper.js | 62 ++++++++++ test/assets/input/scrollable.html | 23 ++++ test/assets/input/textarea.html | 1 + test/test.js | 82 ++++++++++++++ .../test/module/webpage/contextclick-event.js | 5 +- .../test/module/webpage/mouseclick-event.js | 7 +- .../module/webpage/mousedoubleclick-event.js | 3 +- .../test/module/webpage/mousedown-event.js | 3 +- .../test/module/webpage/mousemove-event.js | 3 +- .../test/module/webpage/mouseup-event.js | 3 +- utils/doclint/lint.js | 1 + 17 files changed, 458 insertions(+), 55 deletions(-) create mode 100644 lib/Mouse.js create mode 100644 test/assets/input/mouse-helper.js create mode 100644 test/assets/input/scrollable.html diff --git a/docs/api.md b/docs/api.md index 4f4c477e..05081626 100644 --- a/docs/api.md +++ b/docs/api.md @@ -29,7 +29,7 @@ + [page.$(selector, pageFunction, ...args)](#pageselector-pagefunction-args) + [page.$$(selector, pageFunction, ...args)](#pageselector-pagefunction-args) + [page.addScriptTag(url)](#pageaddscripttagurl) - + [page.click(selector)](#pageclickselector) + + [page.click(selector[, options])](#pageclickselector-options) + [page.close()](#pageclose) + [page.evaluate(pageFunction, ...args)](#pageevaluatepagefunction-args) + [page.evaluateOnInitialized(pageFunction, ...args)](#pageevaluateoninitializedpagefunction-args) @@ -37,10 +37,12 @@ + [page.frames()](#pageframes) + [page.goBack(options)](#pagegobackoptions) + [page.goForward(options)](#pagegoforwardoptions) + + [page.hover(selector)](#pagehoverselector) + [page.httpHeaders()](#pagehttpheaders) + [page.injectFile(filePath)](#pageinjectfilefilepath) + [page.keyboard](#pagekeyboard) + [page.mainFrame()](#pagemainframe) + + [page.mouse](#pagemouse) + [page.navigate(url, options)](#pagenavigateurl-options) + [page.pdf(options)](#pagepdfoptions) + [page.plainText()](#pageplaintext) @@ -67,6 +69,11 @@ + [keyboard.modifiers()](#keyboardmodifiers) + [keyboard.sendCharacter(char)](#keyboardsendcharacterchar) + [keyboard.up(key)](#keyboardupkey) + * [class: Mouse](#class-mouse) + + [mouse.down([options])](#mousedownoptions) + + [mouse.move(x, y)](#mousemovex-y) + + [mouse.press([options])](#mousepressoptions) + + [mouse.up([options])](#mouseupoptions) * [class: Dialog](#class-dialog) + [dialog.accept([promptText])](#dialogacceptprompttext) + [dialog.dismiss()](#dialogdismiss) @@ -76,7 +83,9 @@ + [frame.$(selector, pageFunction, ...args)](#frameselector-pagefunction-args) + [frame.$$(selector, pageFunction, ...args)](#frameselector-pagefunction-args) + [frame.childFrames()](#framechildframes) + + [frame.click(selector[, options])](#frameclickselector-options) + [frame.evaluate(pageFunction, ...args)](#frameevaluatepagefunction-args) + + [frame.hover(selector)](#framehoverselector) + [frame.isDetached()](#frameisdetached) + [frame.isMainFrame()](#frameismainframe) + [frame.name()](#framename) @@ -351,8 +360,11 @@ Shortcut for [page.mainFrame().$$(selector, pageFunction, ...args)](#pageselecto Adds a `` tag to the page with the desired url. Alternatively, javascript could be injected to the page via `page.injectFile` method. -#### page.click(selector) +#### page.click(selector[, options]) - `selector` <[string]> A query selector to search for element to click. If there are multiple elements satisfying the selector, the first will be clicked. +- `options` <[Object]> + - `button` <[string]> `left`, `right`, or `middle`, defaults to `left`. + - `clickCount` <[number]> defaults to 1 - returns: <[Promise]> Promise which resolves when the element matching `selector` is successfully clicked. Promise gets rejected if there's no element matching `selector`. #### page.close() @@ -393,6 +405,10 @@ can not go back, resolves to null. Navigate to the next page in history. +#### page.hover(selector) +- `selector` <[string]> A query selector to search for element to hover. If there are multiple elements satisfying the selector, the first will be hovered. +- returns: <[Promise]> Promise which resolves when the element matching `selector` is successfully hovered. Promise gets rejected if there's no element matching `selector`. + #### page.httpHeaders() - returns: <[Object]> Key-value set of additional http headers which will be sent with every request. @@ -409,6 +425,10 @@ Navigate to the next page in history. Page is guaranteed to have a main frame which persists during navigations. +#### page.mouse + +- returns: <[Mouse]> + #### page.navigate(url, options) - `url` <[string]> URL to navigate page to - `options` <[Object]> Navigation parameters which might have the following properties: @@ -671,6 +691,39 @@ page.keyboard.sendCharacter('嗨'); Dispatches a `keyup` event. +### class: Mouse + +#### mouse.down([options]) +- `options` <[Object]> + - `button` <[string]> `left`, `right`, or `middle`, defaults to `left`. + - `clickCount` <[number]> defaults to 1 +- returns: <[Promise]> + +Dispatches a `mousedown` event. + +#### mouse.move(x, y) +- `x` <[number]> +- `y` <[number]> +- returns: <[Promise]> + +Dispatches a `mousemove` event. + +#### mouse.press([options]) +- `options` <[Object]> + - `button` <[string]> `left`, `right`, or `middle`, defaults to `left`. + - `clickCount` <[number]> defaults to 1 +- returns: <[Promise]> + +Shortcut for [`mouse.down`](#mousedownkey) and [`mouse.up`](#mouseupkey). + +#### mouse.up([options]) +- `options` <[Object]> + - `button` <[string]> `left`, `right`, or `middle`, defaults to `left`. + - `clickCount` <[number]> defaults to 1 +- returns: <[Promise]> + +Dispatches a `mouseup` event. + ### class: Dialog [Dialog] objects are dispatched by page via the ['dialog'](#event-dialog) event. @@ -749,6 +802,13 @@ browser.newPage().then(async page => { #### frame.childFrames() - returns: <[Array]<[Frame]>> +#### frame.click(selector[, options]) +- `selector` <[string]> A query selector to search for element to click. If there are multiple elements satisfying the selector, the first will be clicked. +- `options` <[Object]> + - `button` <[string]> `left`, `right`, or `middle`, defaults to `left`. + - `clickCount` <[number]> defaults to 1 +- returns: <[Promise]> Promise which resolves when the element matching `selector` is successfully clicked. Promise gets rejected if there's no element matching `selector`. + #### frame.evaluate(pageFunction, ...args) - `pageFunction` <[function]> Function to be evaluated in browser context - `...args` <...[string]> Arguments to pass to `pageFunction` @@ -768,6 +828,10 @@ browser.newPage().then(async page => }); ``` +#### frame.hover(selector) +- `selector` <[string]> A query selector to search for element to hover. If there are multiple elements satisfying the selector, the first will be hovered. +- returns: <[Promise]> Promise which resolves when the element matching `selector` is successfully hovered. Promise gets rejected if there's no element matching `selector`. + #### frame.isDetached() - returns: <[boolean]> @@ -1011,3 +1075,4 @@ If there's already a header with name `name`, the header gets overwritten. [Element]: https://developer.mozilla.org/en-US/docs/Web/API/element "Element" [Keyboard]: https://github.com/GoogleChrome/puppeteer/blob/master/docs/api.md#class-keyboard "Keyboard" [Dialog]: https://github.com/GoogleChrome/puppeteer/blob/master/docs/api.md#class-dialog "Dialog" +[Mouse]: https://github.com/GoogleChrome/puppeteer/blob/master/docs/api.md#class-mouse "Mouse" diff --git a/lib/FrameManager.js b/lib/FrameManager.js index 7e8ba374..90ce6c87 100644 --- a/lib/FrameManager.js +++ b/lib/FrameManager.js @@ -20,20 +20,23 @@ let helper = require('./helper'); class FrameManager extends EventEmitter { /** * @param {!Connection} client + * @param {!Mouse} mouse * @return {!Promise} */ - static async create(client) { + static async create(client, mouse) { let mainFramePayload = await client.send('Page.getResourceTree'); - return new FrameManager(client, mainFramePayload.frameTree); + return new FrameManager(client, mainFramePayload.frameTree, mouse); } /** * @param {!Connection} client * @param {!Object} frameTree + * @param {!Mouse} mouse */ - constructor(client, frameTree) { + constructor(client, frameTree, mouse) { super(); this._client = client; + this._mouse = mouse; /** @type {!Map} */ this._frames = new Map(); this._mainFrame = this._addFramesRecursively(null, frameTree); @@ -405,6 +408,39 @@ class Frame { return this._frameManager._evaluateOnFrame(this, expression); } + /** + * @param {string} selector + * @return {!Promise} + */ + async hover(selector) { + let center = await this.evaluate(selector => { + let element = document.querySelector(selector); + if (!element) + return null; + element.scrollIntoViewIfNeeded(); + let rect = element.getBoundingClientRect(); + return { + x: (rect.left + rect.right) / 2, + y: (rect.top + rect.bottom) / 2 + }; + }, selector); + if (!center) + throw new Error('No node found for selector: ' + selector); + await this._frameManager._mouse.move(center.x, center.y); + } + + /** + * @param {string} selector + * @param {!Object=} options + * @return {!Promise} + */ + async click(selector, options) { + await this.hover(selector); + await this._frameManager._mouse.press(options); + // This is a hack for now, to make clicking less race-prone + await this.evaluate(() => new Promise(f => requestAnimationFrame(f))); + } + /** * @param {?Object} framePayload */ diff --git a/lib/Mouse.js b/lib/Mouse.js new file mode 100644 index 00000000..6826b17b --- /dev/null +++ b/lib/Mouse.js @@ -0,0 +1,106 @@ +/** + * Copyright 2017 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +class Mouse { + /** + * @param {!Connection} client + * @param {!Keyboard} keyboard + */ + constructor(client, keyboard) { + this._client = client; + this._keyboard = keyboard; + this._x = 0; + this._y = 0; + this._button = 'none'; + } + + /** + * @param {number} x + * @param {number} y + * @return {!Promise} + */ + async move(x, y) { + this._x = x; + this._y = y; + await this._client.send('Input.dispatchMouseEvent', { + type: 'mouseMoved', + button: this._button, + x, y, + modifiers: this._modifiersMask() + }); + } + + /** + * @param {!Object=} options + */ + async press(options) { + await this.down(options); + await this.up(options); + } + + /** + * @param {!Object=} options + */ + async down(options) { + if (!options) + options = {}; + this._button = (options.button || 'left'); + await this._client.send('Input.dispatchMouseEvent', { + type: 'mousePressed', + button: this._button, + x: this._x, + y: this._y, + modifiers: this._modifiersMask(), + clickCount: (options.clickCount || 1) + }); + } + + /** + * @param {!Object=} options + */ + async up(options) { + if (!options) + options = {}; + this._button = 'none'; + await this._client.send('Input.dispatchMouseEvent', { + type: 'mouseReleased', + button: (options.button || 'left'), + x: this._x, + y: this._y, + modifiers: this._modifiersMask(), + clickCount: (options.clickCount || 1) + }); + } + + /** + * @return {number} + */ + _modifiersMask() { + let modifiers = this._keyboard.modifiers(); + let mask = 0; + if (modifiers.Alt) + mask += 1; + if (modifiers.Control) + mask += 2; + if (modifiers.Meta) + mask += 4; + if (modifiers.Shift) + mask += 8; + return mask; + } +} + +module.exports = Mouse; \ No newline at end of file diff --git a/lib/Page.js b/lib/Page.js index e7485b83..b235a603 100644 --- a/lib/Page.js +++ b/lib/Page.js @@ -23,6 +23,7 @@ let Dialog = require('./Dialog'); let EmulationManager = require('./EmulationManager'); let FrameManager = require('./FrameManager'); let Keyboard = require('./Keyboard'); +let Mouse = require('./Mouse'); let helper = require('./helper'); class Page extends EventEmitter { @@ -40,9 +41,11 @@ class Page extends EventEmitter { ]); let userAgentExpression = helper.evaluationString(() => window.navigator.userAgent); let {result:{value: userAgent}} = await client.send('Runtime.evaluate', { expression: userAgentExpression, returnByValue: true }); - let frameManager = await FrameManager.create(client); + let keyboard = new Keyboard(client); + let mouse = new Mouse(client, keyboard); + let frameManager = await FrameManager.create(client, mouse); let networkManager = new NetworkManager(client, userAgent); - let page = new Page(client, frameManager, networkManager, screenshotTaskQueue); + let page = new Page(client, frameManager, networkManager, screenshotTaskQueue, mouse, keyboard); // Initialize default page size. await page.setViewport({width: 400, height: 300}); return page; @@ -53,8 +56,10 @@ class Page extends EventEmitter { * @param {!FrameManager} frameManager * @param {!NetworkManager} networkManager * @param {!TaskQueue} screenshotTaskQueue + * @param {!Mouse} mouse + * @param {!Keyboard} keyboard */ - constructor(client, frameManager, networkManager, screenshotTaskQueue) { + constructor(client, frameManager, networkManager, screenshotTaskQueue, mouse, keyboard) { super(); this._client = client; this._frameManager = frameManager; @@ -62,7 +67,8 @@ class Page extends EventEmitter { /** @type {!Map} */ this._inPageCallbacks = new Map(); - this._keyboard = new Keyboard(this._client); + this._keyboard = keyboard; + this._mouse = mouse; this._screenshotTaskQueue = screenshotTaskQueue; @@ -527,41 +533,28 @@ class Page extends EventEmitter { return nodeId; } + /** + * @return {!Mouse} + */ + get mouse() { + return this._mouse; + } + /** * @param {string} selector + * @param {!Object} options * @return {!Promise} */ - async click(selector) { - let center = await this.evaluate(selector => { - let node = document.querySelector(selector); - if (!node) - return null; - let rect = node.getBoundingClientRect(); - return { - x: (rect.left + rect.right) / 2, - y: (rect.top + rect.bottom) / 2 - }; - }, selector); - if (!center) - throw new Error('No node found for selector: ' + selector); - let x = Math.round(center.x); - let y = Math.round(center.y); - this._client.send('Input.dispatchMouseEvent', { - type: 'mouseMoved', - x, y - }); - this._client.send('Input.dispatchMouseEvent', { - type: 'mousePressed', - button: 'left', - x, y, - clickCount: 1 - }); - await this._client.send('Input.dispatchMouseEvent', { - type: 'mouseReleased', - button: 'left', - x, y, - clickCount: 1 - }); + async click(selector, options) { + await this.mainFrame().click(selector, options); + } + + /** + * @param {string} selector + * @param {!Promise} + */ + async hover(selector) { + await this.mainFrame().hover(selector); } /** diff --git a/phantom_shim/WebPage.js b/phantom_shim/WebPage.js index 9a934cbe..723a5e70 100644 --- a/phantom_shim/WebPage.js +++ b/phantom_shim/WebPage.js @@ -77,6 +77,9 @@ class WebPage { Backspace: ['Backspace'], Cut: ['Cut'], Paste: ['Paste'] + }, + modifier: { + shift: 'Shift' } }; } @@ -401,6 +404,8 @@ class WebPage { sendEvent(eventType, ...args) { if (eventType.startsWith('key')) this._sendKeyboardEvent.apply(this, arguments); + else + this._sendMouseEvent.apply(this, arguments); } /** @@ -449,6 +454,40 @@ class WebPage { } } + /** + * @param {string} eventType + * @param {number} x + * @param {number} y + * @param {string|undefined} button + * @param {number|undefined} modifier + */ + _sendMouseEvent(eventType, x, y, button, modifier) { + if (modifier) + await(this._page.keyboard.down(modifier)); + await(this._page.mouse.move(x, y)); + switch (eventType) { + case 'mousemove': + break; + case 'mousedown': + await(this._page.mouse.down({button})); + break; + case 'mouseup': + await(this._page.mouse.up({button})); + break; + case 'doubleclick': + await(this._page.mouse.press({button})); + await(this._page.mouse.press({button, clickCount: 2})); + break; + case 'click': + await(this._page.mouse.press({button})); + break; + case 'contextmenu': + await(this._page.mouse.press({button: 'right'})); + break; + } + if (modifier) + await(this._page.keyboard.up(modifier)); + } /** * @param {string} html * @param {function()=} callback diff --git a/test/assets/input/button.html b/test/assets/input/button.html index 8b59f762..d4c6e13f 100644 --- a/test/assets/input/button.html +++ b/test/assets/input/button.html @@ -4,6 +4,7 @@ Button test + + + + \ No newline at end of file diff --git a/test/assets/input/textarea.html b/test/assets/input/textarea.html index b6360739..6d5be760 100644 --- a/test/assets/input/textarea.html +++ b/test/assets/input/textarea.html @@ -5,6 +5,7 @@ +