feat(accessibility): snapshot the accessibility tree (#3470)

This adds `page.accessibility.snapshot()`. It serializes and returns the accessibility tree for the page. By default, uninteresting nodes are filtered out of the snapshot.

fixes #2033
This commit is contained in:
Joel Einbinder 2018-11-01 18:54:51 -07:00 committed by Andrey Lushnikov
parent eca3c6bed2
commit 9ba3261571
6 changed files with 704 additions and 1 deletions

View File

@ -90,6 +90,7 @@
* [page.$$eval(selector, pageFunction[, ...args])](#pageevalselector-pagefunction-args)
* [page.$eval(selector, pageFunction[, ...args])](#pageevalselector-pagefunction-args-1)
* [page.$x(expression)](#pagexexpression)
* [page.accessibility](#pageaccessibility)
* [page.addScriptTag(options)](#pageaddscripttagoptions)
* [page.addStyleTag(options)](#pageaddstyletagoptions)
* [page.authenticate(credentials)](#pageauthenticatecredentials)
@ -156,6 +157,8 @@
* [worker.evaluateHandle(pageFunction, ...args)](#workerevaluatehandlepagefunction-args)
* [worker.executionContext()](#workerexecutioncontext)
* [worker.url()](#workerurl)
- [class: Accessibility](#class-accessibility)
* [accessibility.snapshot([options])](#accessibilitysnapshotoptions)
- [class: Keyboard](#class-keyboard)
* [keyboard.down(key[, options])](#keyboarddownkey-options)
* [keyboard.press(key[, options])](#keyboardpresskey-options)
@ -1047,6 +1050,9 @@ The method evaluates the XPath expression.
Shortcut for [page.mainFrame().$x(expression)](#framexexpression)
#### page.accessibility
- returns: <[Accessibility]>
#### page.addScriptTag(options)
- `options` <[Object]>
- `url` <[string]> URL of a script to be added.
@ -1982,6 +1988,79 @@ Shortcut for [(await worker.executionContext()).evaluateHandle(pageFunction, ...
#### worker.url()
- returns: <[string]>
### class: Accessibility
The Accessibility class provides methods for inspecting Chromium's accessibility tree. The accessibility tree is used by assistive technology such as [screen readers](https://en.wikipedia.org/wiki/Screen_reader).
Accessibility is a very platform-specific thing. On different platforms, there are different screen readers that might have wildly different output.
Blink - Chrome's rendering engine - has a concept of "accessibility tree", which is than translated into different platform-specific APIs. Accessibility namespace gives users
access to the Blink Accessibility Tree.
Most of the accessibility tree gets filtered out when converting from Blink AX Tree to Platform-specific AX-Tree or by screen readers themselves. By default, Puppeteer tries to approximate this filtering, exposing only the "interesting" nodes of the tree.
#### accessibility.snapshot([options])
- `options` <[Object]>
- `interestingOnly` <[boolean]> Prune uninteresting nodes from the tree. Defaults to `true`.
- returns: <[Promise]<[AXNode]>> Returns an AXNode object with the following properties:
- `role` <[string]> The [role](https://www.w3.org/TR/wai-aria/#usage_intro).
- `name` <[string]> A human readable name for the node.
- `value` <[string]|[number]> The current value of the node.
- `description` <[string]> An additional human readable description of the node.
- `keyshortcuts` <[string]> Keyboard shortcuts associated with this node.
- `roledescription` <[string]> A human readable alternative to the role.
- `valuetext` <[string]> A description of the current value.
- `disabled` <[boolean]> Whether the node is disabled.
- `expanded` <[boolean]> Whether the node is expanded or collapsed.
- `focused` <[boolean]> Whether the node is focused.
- `modal` <[boolean]> Whether the node is [modal](https://en.wikipedia.org/wiki/Modal_window).
- `multiline` <[boolean]> Whether the node text input supports multiline.
- `multiselectable` <[boolean]> Whether more than one child can be selected.
- `readonly` <[boolean]> Whether the node is read only.
- `required` <[boolean]> Whether the node is required.
- `selected` <[boolean]> Whether the node is selected in its parent node.
- `checked` <[boolean]|[string]> Whether the checkbox is checked, or "mixed".
- `pressed` <[boolean]|[string]> Whether the toggle button is checked, or "mixed".
- `level` <[number]> The level of a heading.
- `valuemin` <[number]> The minimum value in a node.
- `valuemax` <[number]> The maximum value in a node.
- `autocomplete` <[string]> What kind of autocomplete is supported by a control.
- `haspopup` <[string]> What kind of popup is currently being shown for a node.
- `invalid` <[string]> Whether and in what way this node's value is invalid.
- `orientation` <[string]> Whether the node is oriented horizontally or vertically.
- `children` <[Array]<[AXNode]>> Child nodes of this node, if any.
Captures the current state of the accessibility tree. The returned object represents the root accessible node of the page.
> **NOTE** The Chromium accessibility tree contains nodes that go unused on most platforms and by
most screen readers. Puppeteer will discard them as well for an easier to process tree,
unless `interestingOnly` is set to `false`.
An example of dumping the entire accessibility tree:
```js
const snapshot = await page.accessibility.snapshot();
console.log(snapshot);
```
An example of logging the focused node's name:
```js
const snapshot = await page.accessibility.snapshot();
const node = findFocusedNode(snapshot);
console.log(node && node.name);
function findFocusedNode(node) {
if (node.focused)
return node;
for (const child of node.children || []) {
const foundNode = findFocusedNode(child);
return foundNode;
}
return null;
}
```
### class: Keyboard
Keyboard provides an api for managing a virtual keyboard. The high level api is [`keyboard.type`](#keyboardtypetext-options), which takes raw characters and generates proper keydown, keypress/input, and keyup events on your page.
@ -3443,3 +3522,5 @@ TimeoutError is emitted whenever certain operations are terminated due to timeou
[UnixTime]: https://en.wikipedia.org/wiki/Unix_time "Unix Time"
[SecurityDetails]: #class-securitydetails "SecurityDetails"
[Worker]: #class-worker "Worker"
[Accessibility]: #class-accessibility "Accessibility"
[AXNode]: #accessibilitysnapshotoptions "AXNode"

393
lib/Accessibility.js Normal file
View File

@ -0,0 +1,393 @@
/**
* Copyright 2018 Google Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the 'License');
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an 'AS IS' BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
const {helper} = require('./helper');
/**
* @typedef {Object} SerializedAXNode
* @property {string} role
*
* @property {string=} name
* @property {string|number=} value
* @property {string=} description
*
* @property {string=} keyshortcuts
* @property {string=} roledescription
* @property {string=} valuetext
*
* @property {boolean=} disabled
* @property {boolean=} expanded
* @property {boolean=} focused
* @property {boolean=} modal
* @property {boolean=} multiline
* @property {boolean=} multiselectable
* @property {boolean=} readonly
* @property {boolean=} required
* @property {boolean=} selected
*
* @property {boolean|"mixed"=} checked
* @property {boolean|"mixed"=} pressed
*
* @property {number=} level
* @property {number=} valuemin
* @property {number=} valuemax
*
* @property {string=} autocomplete
* @property {string=} haspopup
* @property {string=} invalid
* @property {string=} orientation
*
* @property {Array<SerializedAXNode>=} children
*/
class Accessibility {
/**
* @param {!Puppeteer.CDPSession} client
*/
constructor(client) {
this._client = client;
}
/**
* @param {{interestingOnly?: boolean}=} options
* @return {!Promise<!SerializedAXNode>}
*/
async snapshot(options = {}) {
const {interestingOnly = true} = options;
const {nodes} = await this._client.send('Accessibility.getFullAXTree');
const root = AXNode.createTree(nodes);
if (!interestingOnly)
return serializeTree(root)[0];
/** @type {!Set<!AXNode>} */
const interestingNodes = new Set();
collectInterestingNodes(interestingNodes, root, false);
return serializeTree(root, interestingNodes)[0];
}
}
/**
* @param {!Set<!AXNode>} collection
* @param {!AXNode} node
* @param {boolean} insideControl
*/
function collectInterestingNodes(collection, node, insideControl) {
if (node.isInteresting(insideControl))
collection.add(node);
if (node.isLeafNode())
return;
insideControl = insideControl || node.isControl();
for (const child of node._children)
collectInterestingNodes(collection, child, insideControl);
}
/**
* @param {!AXNode} node
* @param {!Set<!AXNode>=} whitelistedNodes
* @return {!Array<!SerializedAXNode>}
*/
function serializeTree(node, whitelistedNodes) {
/** @type {!Array<!SerializedAXNode>} */
const children = [];
for (const child of node._children)
children.push(...serializeTree(child, whitelistedNodes));
if (whitelistedNodes && !whitelistedNodes.has(node))
return children;
const serializedNode = node.serialize();
if (children.length)
serializedNode.children = children;
return [serializedNode];
}
class AXNode {
/**
* @param {!Protocol.Accessibility.AXNode} payload
*/
constructor(payload) {
this._payload = payload;
/** @type {!Array<!AXNode>} */
this._children = [];
this._richlyEditable = false;
this._editable = false;
this._focusable = false;
this._expanded = false;
this._name = this._payload.name ? this._payload.name.value : '';
this._role = this._payload.role ? this._payload.role.value : 'Unknown';
this._cachedHasFocusableChild;
for (const property of this._payload.properties || []) {
if (property.name === 'editable') {
this._richlyEditable = property.value.value === 'richtext';
this._editable = true;
}
if (property.name === 'focusable')
this._focusable = property.value.value;
if (property.name === 'expanded')
this._expanded = property.value.value;
}
}
/**
* @return {boolean}
*/
_isPlainTextField() {
if (this._richlyEditable)
return false;
if (this._editable)
return true;
return this._role === 'textbox' || this._role === 'ComboBox' || this._role === 'searchbox';
}
/**
* @return {boolean}
*/
_isTextOnlyObject() {
const role = this._role;
return (role === 'LineBreak' || role === 'text' ||
role === 'InlineTextBox');
}
/**
* @return {boolean}
*/
_hasFocusableChild() {
if (this._cachedHasFocusableChild === undefined) {
this._cachedHasFocusableChild = false;
for (const child of this._children) {
if (child._focusable || child._hasFocusableChild()) {
this._cachedHasFocusableChild = true;
break;
}
}
}
return this._cachedHasFocusableChild;
}
/**
* @return {boolean}
*/
isLeafNode() {
if (!this._children.length)
return true;
// These types of objects may have children that we use as internal
// implementation details, but we want to expose them as leaves to platform
// accessibility APIs because screen readers might be confused if they find
// any children.
if (this._isPlainTextField() || this._isTextOnlyObject())
return true;
// Roles whose children are only presentational according to the ARIA and
// HTML5 Specs should be hidden from screen readers.
// (Note that whilst ARIA buttons can have only presentational children, HTML5
// buttons are allowed to have content.)
switch (this._role) {
case 'doc-cover':
case 'graphics-symbol':
case 'img':
case 'Meter':
case 'scrollbar':
case 'slider':
case 'separator':
case 'progressbar':
return true;
default:
break;
}
// Here and below: Android heuristics
if (this._hasFocusableChild())
return false;
if (this._focusable && this._name)
return true;
if (this._role === 'heading' && this._name)
return true;
return false;
}
/**
* @return {boolean}
*/
isControl() {
switch (this._role) {
case 'button':
case 'checkbox':
case 'ColorWell':
case 'combobox':
case 'DisclosureTriangle':
case 'listbox':
case 'menu':
case 'menubar':
case 'menuitem':
case 'menuitemcheckbox':
case 'menuitemradio':
case 'radio':
case 'scrollbar':
case 'searchbox':
case 'slider':
case 'spinbutton':
case 'switch':
case 'tab':
case 'textbox':
case 'tree':
return true;
default:
return false;
}
}
/**
* @param {boolean} insideControl
* @return {boolean}
*/
isInteresting(insideControl) {
const role = this._role;
if (role === 'Ignored')
return false;
if (this._focusable || this._richlyEditable)
return true;
// If it's not focusable but has a control role, then it's interesting.
if (this.isControl())
return true;
// A non focusable child of a control is not interesting
if (insideControl)
return false;
return this.isLeafNode() && !!this._name;
}
/**
* @return {!SerializedAXNode}
*/
serialize() {
/** @type {!Map<string, number|string|boolean>} */
const properties = new Map();
for (const property of this._payload.properties || [])
properties.set(property.name.toLowerCase(), property.value.value);
if (this._payload.name)
properties.set('name', this._payload.name.value);
if (this._payload.value)
properties.set('value', this._payload.value.value);
if (this._payload.description)
properties.set('description', this._payload.description.value);
/** @type {SerializedAXNode} */
const node = {
role: this._role
};
/** @type {!Array<keyof SerializedAXNode>} */
const userStringProperties = [
'name',
'value',
'description',
'keyshortcuts',
'roledescription',
'valuetext',
];
for (const userStringProperty of userStringProperties) {
if (!properties.has(userStringProperty))
continue;
node[userStringProperty] = properties.get(userStringProperty);
}
/** @type {!Array<keyof SerializedAXNode>} */
const booleanProperties = [
'disabled',
'expanded',
'focused',
'modal',
'multiline',
'multiselectable',
'readonly',
'required',
'selected',
];
for (const booleanProperty of booleanProperties) {
// WebArea's treat focus differently than other nodes. They report whether their frame has focus,
// not whether focus is specifically on the root node.
if (booleanProperty === 'focused' && this._role === 'WebArea')
continue;
const value = properties.get(booleanProperty);
if (!value)
continue;
node[booleanProperty] = value;
}
/** @type {!Array<keyof SerializedAXNode>} */
const tristateProperties = [
'checked',
'pressed',
];
for (const tristateProperty of tristateProperties) {
if (!properties.has(tristateProperty))
continue;
const value = properties.get(tristateProperty);
node[tristateProperty] = value === 'mixed' ? 'mixed' : value === 'true' ? true : false;
}
/** @type {!Array<keyof SerializedAXNode>} */
const numericalProperties = [
'level',
'valuemax',
'valuemin',
];
for (const numericalProperty of numericalProperties) {
if (!properties.has(numericalProperty))
continue;
node[numericalProperty] = properties.get(numericalProperty);
}
/** @type {!Array<keyof SerializedAXNode>} */
const tokenProperties = [
'autocomplete',
'haspopup',
'invalid',
'orientation',
];
for (const tokenProperty of tokenProperties) {
const value = properties.get(tokenProperty);
if (!value || value === 'false')
continue;
node[tokenProperty] = value;
}
return node;
}
/**
* @param {!Array<!Protocol.Accessibility.AXNode>} payloads
* @return {!AXNode}
*/
static createTree(payloads) {
/** @type {!Map<string, !AXNode>} */
const nodeById = new Map();
for (const payload of payloads)
nodeById.set(payload.nodeId, new AXNode(payload));
for (const node of nodeById.values()) {
for (const childId of node._payload.childIds || [])
node._children.push(nodeById.get(childId));
}
return nodeById.values().next().value;
}
}
module.exports = {Accessibility};
helper.tracePublicAPI(Accessibility);

View File

@ -27,7 +27,7 @@ const {helper, debugError, assert} = require('./helper');
const {Coverage} = require('./Coverage');
const {Worker} = require('./Worker');
const {createJSHandle} = require('./ExecutionContext');
const {Accessibility} = require('./Accessibility');
const writeFileAsync = helper.promisify(fs.writeFile);
class Page extends EventEmitter {
@ -78,6 +78,7 @@ class Page extends EventEmitter {
this._keyboard = new Keyboard(client);
this._mouse = new Mouse(client, this._keyboard);
this._touchscreen = new Touchscreen(client, this._keyboard);
this._accessibility = new Accessibility(client);
this._networkManager = new NetworkManager(client);
/** @type {!FrameManager} */
this._frameManager = new FrameManager(client, frameTree, this, this._networkManager);
@ -221,6 +222,13 @@ class Page extends EventEmitter {
return this._tracing;
}
/**
* @return {!Accessibility}
*/
get accessibility() {
return this._accessibility;
}
/**
* @return {!Array<Puppeteer.Frame>}
*/

219
test/accessibility.spec.js Normal file
View File

@ -0,0 +1,219 @@
/**
* Copyright 2018 Google Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
module.exports.addTests = function({testRunner, expect}) {
const {describe, xdescribe, fdescribe} = testRunner;
const {it, fit, xit} = testRunner;
const {beforeAll, beforeEach, afterAll, afterEach} = testRunner;
describe('Accessibility', function() {
it('should work', async function({page}) {
await page.setContent(`
<head>
<title>Accessibility Test</title>
</head>
<body>
<div>Hello World</div>
<h1>Inputs</h1>
<input placeholder="Empty input" autofocus />
<input placeholder="readonly input" readonly />
<input placeholder="disabled input" disabled />
<input aria-label="Input with whitespace" value=" " />
<input value="value only" />
<input aria-placeholder="placeholder" value="and a value" />
<div aria-hidden="true" id="desc">This is a description!</div>
<input aria-placeholder="placeholder" value="and a value" aria-describedby="desc" />
<select>
<option>First Option</option>
<option>Second Option</option>
</select>
</body>`);
expect(await page.accessibility.snapshot()).toEqual({
role: 'WebArea',
name: 'Accessibility Test',
children: [
{role: 'text', name: 'Hello World'},
{role: 'heading', name: 'Inputs', level: 1},
{role: 'textbox', name: 'Empty input', focused: true},
{role: 'textbox', name: 'readonly input', readonly: true},
{role: 'textbox', name: 'disabled input', disabled: true},
{role: 'textbox', name: 'Input with whitespace', value: ' '},
{role: 'textbox', name: '', value: 'value only'},
{role: 'textbox', name: 'placeholder', value: 'and a value'},
{role: 'textbox', name: 'placeholder', value: 'and a value', description: 'This is a description!'},
{role: 'combobox', name: '', value: 'First Option', children: [
{role: 'menuitem', name: 'First Option', selected: true},
{role: 'menuitem', name: 'Second Option'}]}]
});
});
it('should report uninteresting nodes', async function({page}) {
await page.setContent(`<textarea autofocus>hi</textarea>`);
expect(findFocusedNode(await page.accessibility.snapshot({interestingOnly: false}))).toEqual({
role: 'textbox',
name: '',
value: 'hi',
focused: true,
multiline: true,
children: [{
role: 'GenericContainer',
name: '',
children: [{
role: 'text', name: 'hi'
}]
}]
});
});
describe('filtering children of leaf nodes', function() {
it('should not report text nodes inside controls', async function({page}) {
await page.setContent(`
<div role="tablist">
<div role="tab" aria-selected="true"><b>Tab1</b></div>
<div role="tab">Tab2</div>
</div>`);
expect(await page.accessibility.snapshot()).toEqual({
role: 'WebArea',
name: '',
children: [{
role: 'tab',
name: 'Tab1',
selected: true
}, {
role: 'tab',
name: 'Tab2'
}]
});
});
it('rich text editable fields should have children', async function({page}) {
await page.setContent(`
<div contenteditable="true">
Edit this image: <img src="fakeimage.png" alt="my fake image">
</div>`);
const snapshot = await page.accessibility.snapshot();
expect(snapshot.children[0]).toEqual({
role: 'GenericContainer',
name: '',
value: 'Edit this image: ',
children: [{
role: 'text',
name: 'Edit this image:'
}, {
role: 'img',
name: 'my fake image'
}]
});
});
it('rich text editable fields with role should have children', async function({page}) {
await page.setContent(`
<div contenteditable="true" role='textbox'>
Edit this image: <img src="fakeimage.png" alt="my fake image">
</div>`);
const snapshot = await page.accessibility.snapshot();
expect(snapshot.children[0]).toEqual({
role: 'textbox',
name: '',
value: 'Edit this image: ',
children: [{
role: 'text',
name: 'Edit this image:'
}, {
role: 'img',
name: 'my fake image'
}]
});
});
it('plain text field with role should not have children', async function({page}) {
await page.setContent(`
<div contenteditable="plaintext-only" role='textbox'>Edit this image:<img src="fakeimage.png" alt="my fake image"></div>`);
const snapshot = await page.accessibility.snapshot();
expect(snapshot.children[0]).toEqual({
role: 'textbox',
name: '',
value: 'Edit this image:'
});
});
it('plain text field without role should not have content', async function({page}) {
await page.setContent(`
<div contenteditable="plaintext-only">Edit this image:<img src="fakeimage.png" alt="my fake image"></div>`);
const snapshot = await page.accessibility.snapshot();
expect(snapshot.children[0]).toEqual({
role: 'GenericContainer',
name: ''
});
});
it('plain text field with tabindex and without role should not have content', async function({page}) {
await page.setContent(`
<div contenteditable="plaintext-only">Edit this image:<img src="fakeimage.png" alt="my fake image"></div>`);
const snapshot = await page.accessibility.snapshot();
expect(snapshot.children[0]).toEqual({
role: 'GenericContainer',
name: ''
});
});
it('non editable textbox with role and tabIndex and label should not have children', async function({page}) {
await page.setContent(`
<div role="textbox" tabIndex=0 aria-checked="true" aria-label="my favorite textbox">
this is the inner content
<img alt="yo" src="fakeimg.png">
</div>`);
const snapshot = await page.accessibility.snapshot();
expect(snapshot.children[0]).toEqual({
role: 'textbox',
name: 'my favorite textbox',
value: 'this is the inner content '
});
});
it('checkbox with and tabIndex and label should not have children', async function({page}) {
await page.setContent(`
<div role="checkbox" tabIndex=0 aria-checked="true" aria-label="my favorite checkbox">
this is the inner content
<img alt="yo" src="fakeimg.png">
</div>`);
const snapshot = await page.accessibility.snapshot();
expect(snapshot.children[0]).toEqual({
role: 'checkbox',
name: 'my favorite checkbox',
checked: true
});
});
it('checkbox without label should not have children', async function({page}) {
await page.setContent(`
<div role="checkbox" aria-checked="true">
this is the inner content
<img alt="yo" src="fakeimg.png">
</div>`);
const snapshot = await page.accessibility.snapshot();
expect(snapshot.children[0]).toEqual({
role: 'checkbox',
name: 'this is the inner content yo',
checked: true
});
});
});
function findFocusedNode(node) {
if (node.focused)
return node;
for (const child of node.children || []) {
const focusedChild = findFocusedNode(child);
if (focusedChild)
return focusedChild;
}
return null;
}
});
};

View File

@ -146,6 +146,7 @@ describe('Browser', function() {
// Page-level tests that are given a browser, a context and a page.
// Each test is launched in a new browser context.
require('./CDPSession.spec.js').addTests({testRunner, expect});
require('./accessibility.spec.js').addTests({testRunner, expect});
require('./browser.spec.js').addTests({testRunner, expect, headless});
require('./cookies.spec.js').addTests({testRunner, expect});
require('./coverage.spec.js').addTests({testRunner, expect});

View File

@ -20,6 +20,7 @@ const Documentation = require('./Documentation');
const Message = require('../Message');
const EXCLUDE_CLASSES = new Set([
'AXNode',
'CSSCoverage',
'Connection',
'CustomError',