feat(Browser): introduce Browser.pages() (#554)

This patch:
- introduces Target class that represents any inspectable target, such as service worker or page
- emits events when targets come and go
- introduces target.page() to instantiate a page from a target

Fixes #386, fixes #443.

docs/api.md

@@ -13,9 +13,14 @@
   * [puppeteer.executablePath()](#puppeteerexecutablepath)
   * [puppeteer.launch([options])](#puppeteerlaunchoptions)
 - [class: Browser](#class-browser)
+  * [event: 'targetchanged'](#event-targetchanged)
+  * [event: 'targetcreated'](#event-targetcreated)
+  * [event: 'targetdestroyed'](#event-targetdestroyed)
   * [browser.close()](#browserclose)
   * [browser.disconnect()](#browserdisconnect)
   * [browser.newPage()](#browsernewpage)
+  * [browser.pages()](#browserpages)
+  * [browser.targets()](#browsertargets)
   * [browser.version()](#browserversion)
   * [browser.wsEndpoint()](#browserwsendpoint)
 - [class: Page](#class-page)
@@ -175,6 +180,10 @@
   * [response.status](#responsestatus)
   * [response.text()](#responsetext)
   * [response.url](#responseurl)
+- [class: Target](#class-target)
+  * [target.page()](#targetpage)
+  * [target.type()](#targettype)
+  * [target.url()](#targeturl)
 
 <!-- tocstop -->
 
@@ -278,6 +287,21 @@ puppeteer.launch().then(async browser => {
 });
 ```
 
+#### event: 'targetchanged'
+- <[Target]>
+
+Emitted when the url of a target changes.
+
+#### event: 'targetcreated'
+- <[Target]>
+
+Emitted when a target is created, for example when a new page is opened by [`window.open`](https://developer.mozilla.org/en-US/docs/Web/API/Window/open) or [`browser.newPage`](#browsernewpage).
+
+#### event: 'targetdestroyed'
+- <[Target]>
+
+Emitted when a target is destroyed, for example when a page is closed.
+
 #### browser.close()
 - returns: <[Promise]>
 
@@ -290,6 +314,12 @@ Disconnects Puppeteer from the browser, but leaves the Chromium process running.
 #### browser.newPage()
 - returns: <[Promise]<[Page]>> Promise which resolves to a new [Page] object.
 
+#### browser.pages()
+- returns: <[Promise]<[Array]<[Page]>>> Promise which resolves to an array of all open pages.
+
+#### browser.targets()
+- returns: <[Array]<[Target]>> An array of all active targets.
+
 #### browser.version()
 - returns: <[Promise]<[string]>> For headless Chromium, this is similar to `HeadlessChrome/61.0.3153.0`. For non-headless, this is similar to `Chrome/61.0.3153.0`.
 
@@ -1896,6 +1926,21 @@ Contains the status code of the response (e.g., 200 for a success).
 
 Contains the URL of the response.
 
+### class: Target
+
+#### target.page()
+- returns: <[Promise]<[Page]>>
+
+If the target is not of type `"page"`, returns `null`.
+
+#### target.type()
+- returns: <[string]>
+
+Identifies what kind of target this is. Can be `"page"`, `"service_worker"`, or `"other"`.
+
+#### target.url()
+- returns: <[string]>
+
 [Array]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array "Array"
 [boolean]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Data_structures#Boolean_type "Boolean"
 [Buffer]: https://nodejs.org/api/buffer.html#buffer_class_buffer "Buffer"
@@ -1927,3 +1972,4 @@ Contains the URL of the response.
 [UIEvent.detail]: https://developer.mozilla.org/en-US/docs/Web/API/UIEvent/detail "UIEvent.detail"
 [Serializable]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/JSON/stringify#Description "Serializable"
 [Touchscreen]: #class-touchscreen "Touchscreen"
+[Target]: #class-target "Target"

lib/Browser.js

@@ -31,6 +31,54 @@ class Browser extends EventEmitter {
     this._screenshotTaskQueue = new TaskQueue();
     this._connection = connection;
     this._closeCallback = closeCallback || new Function();
+    /** @type {Map<string, Target>} */
+    this._targets = new Map();
+    this._connection.on('Target.targetCreated', this._targetCreated.bind(this));
+    this._connection.on('Target.targetDestroyed', this._targetDestroyed.bind(this));
+    this._connection.on('Target.targetInfoChanged', this._targetInfoChanged.bind(this));
+  }
+
+  /**
+   * @param {!Puppeteer.Connection} connection
+   * @param {boolean} ignoreHTTPSErrors
+   * @param {function()=} closeCallback
+   */
+  static async create(connection, ignoreHTTPSErrors, closeCallback) {
+    const browser =  new Browser(connection, ignoreHTTPSErrors, closeCallback);
+    await connection.send('Target.setDiscoverTargets', {discover: true});
+    return browser;
+  }
+
+  /**
+   * @param {{targetInfo: !Target.TargetInfo}} event
+   */
+  async _targetCreated(event) {
+    const target = new Target(this, event.targetInfo);
+    console.assert(!this._targets.has(event.targetInfo.targetId), 'Target should not exist before targetCreated');
+    this._targets.set(event.targetInfo.targetId, target);
+
+    if (await target._initializedPromise)
+      this.emit(Browser.Events.TargetCreated, target);
+  }
+
+  /**
+   * @param {{targetId: string}} event
+   */
+  async _targetDestroyed(event) {
+    const target = this._targets.get(event.targetId);
+    target._initializedCallback(false);
+    this._targets.delete(event.targetId);
+    if (await target._initializedPromise)
+      this.emit(Browser.Events.TargetDestroyed, target);
+  }
+
+  /**
+   * @param {{targetInfo: !Target.TargetInfo}} event
+   */
+  _targetInfoChanged(event) {
+    const target = this._targets.get(event.targetInfo.targetId);
+    console.assert(target, 'target should exist before targetInfoChanged');
+    target._targetInfoChanged(event.targetInfo);
   }
 
   /**
@@ -45,8 +93,25 @@ class Browser extends EventEmitter {
    */
   async newPage() {
     const {targetId} = await this._connection.send('Target.createTarget', {url: 'about:blank'});
-    const client = await this._connection.createSession(targetId);
+    const target = await this._targets.get(targetId);
-    return await Page.create(client, this._ignoreHTTPSErrors, this._appMode, this._screenshotTaskQueue);
+    console.assert(await target._initializedPromise, 'Failed to create target for page');
+    const page = await target.page();
+    return page;
+  }
+
+  /**
+   * @return {!Array<!Target>}
+   */
+  targets() {
+    return Array.from(this._targets.values()).filter(target => target._isInitialized);
+  }
+
+  /**
+   * @return {!Promise<!Array<!Page>>}
+   */
+  async pages() {
+    const pages = await Promise.all(this.targets().map(target => target.page()));
+    return pages.filter(page => !!page);
   }
 
   /**
@@ -67,6 +132,13 @@ class Browser extends EventEmitter {
   }
 }
 
+/** @enum {string} */
+Browser.Events = {
+  TargetCreated: 'targetcreated',
+  TargetDestroyed: 'targetdestroyed',
+  TargetChanged: 'targetchanged'
+};
+
 helper.tracePublicAPI(Browser);
 
 class TaskQueue {
@@ -84,4 +156,77 @@ class TaskQueue {
     return result;
   }
 }
+
+class Target {
+  /**
+   * @param {!Browser} browser
+   * @param {!Target.TargetInfo} targetInfo
+   */
+  constructor(browser, targetInfo) {
+    this._browser = browser;
+    this._targetInfo = targetInfo;
+    /** @type {?Promise<!Page>} */
+    this._pagePromise = null;
+    this._initializedPromise = new Promise(fulfill => this._initializedCallback = fulfill);
+    this._isInitialized = this._targetInfo.type !== 'page' || this._targetInfo.url !== '';
+    if (this._isInitialized)
+      this._initializedCallback(true);
+  }
+
+  /**
+   * @return {!Promise<?Page>}
+   */
+  async page() {
+    if (this._targetInfo.type === 'page' && !this._pagePromise) {
+      this._pagePromise = this._browser._connection.createSession(this._targetInfo.targetId)
+          .then(client => Page.create(client, this._browser._ignoreHTTPSErrors, this._browser._appMode, this._browser._screenshotTaskQueue));
+    }
+    return this._pagePromise;
+  }
+
+  /**
+   * @return {string}
+   */
+  url() {
+    return this._targetInfo.url;
+  }
+
+  /**
+   * @return {"page"|"service_worker"|"other"}
+   */
+  type() {
+    const type = this._targetInfo.type;
+    if (type === 'page' || type === 'service_worker')
+      return type;
+    return 'other';
+  }
+
+  /**
+   * @param {!Target.TargetInfo} targetInfo
+   */
+  _targetInfoChanged(targetInfo) {
+    const previousURL = this._targetInfo.url;
+    this._targetInfo = targetInfo;
+
+    if (!this._isInitialized && (this._targetInfo.type !== 'page' || this._targetInfo.url !== '')) {
+      this._isInitialized = true;
+      this._initializedCallback(true);
+      return;
+    }
+
+    if (previousURL !== targetInfo.url)
+      this._browser.emit(Browser.Events.TargetChanged, this);
+  }
+}
+helper.tracePublicAPI(Target);
+
+/**
+ * @typedef {Object} Target.TargetInfo
+ * @property {string} type
+ * @property {string} targetId
+ * @property {string} title
+ * @property {string} url
+ * @property {boolean} attached
+ */
+
 module.exports = { Browser, TaskQueue };

lib/FrameManager.js

@@ -25,9 +25,10 @@ const readFileAsync = helper.promisify(fs.readFile);
 class FrameManager extends EventEmitter {
   /**
    * @param {!Puppeteer.Session} client
+   * @param {{frame: Object, childFrames: ?Array}} frameTree
    * @param {!Puppeteer.Page} page
    */
-  constructor(client, page) {
+  constructor(client, frameTree, page) {
     super();
     this._client = client;
     this._page = page;
@@ -40,6 +41,22 @@ class FrameManager extends EventEmitter {
     this._client.on('Page.frameNavigated', event => this._onFrameNavigated(event.frame));
     this._client.on('Page.frameDetached', event => this._onFrameDetached(event.frameId));
     this._client.on('Runtime.executionContextCreated', event => this._onExecutionContextCreated(event.context));
+
+    this._handleFrameTree(frameTree);
+  }
+
+  /**
+   * @param {{frame: Object, childFrames: ?Array}} frameTree
+   */
+  _handleFrameTree(frameTree) {
+    if (frameTree.frame.parentId)
+      this._onFrameAttached(frameTree.frame.id, frameTree.frame.parentId);
+    this._onFrameNavigated(frameTree.frame);
+    if (!frameTree.childFrames)
+      return;
+
+    for (const child of frameTree.childFrames)
+      this._handleFrameTree(child);
   }
 
   /**

lib/Launcher.js

@@ -134,7 +134,7 @@ class Launcher {
       const connectionDelay = options.slowMo || 0;
       const browserWSEndpoint = await waitForWSEndpoint(chromeProcess, options.timeout || 30 * 1000);
       connection = await Connection.create(browserWSEndpoint, connectionDelay);
-      return new Browser(connection, options, killChrome);
+      return Browser.create(connection, options, killChrome);
     } catch (e) {
       killChrome();
       throw e;
@@ -179,7 +179,7 @@ class Launcher {
    */
   static async connect(options = {}) {
     const connection = await Connection.create(options.browserWSEndpoint);
-    return new Browser(connection, options, () => connection.send('Browser.close'));
+    return Browser.create(connection, options, () => connection.send('Browser.close'));
   }
 }
 

lib/Page.js

@@ -37,35 +37,39 @@ class Page extends EventEmitter {
    * @return {!Promise<!Page>}
    */
   static async create(client, ignoreHTTPSErrors, appMode, screenshotTaskQueue) {
+
+    await client.send('Page.enable');
+    const {frameTree} = await client.send('Page.getResourceTree');
+    const page = new Page(client, frameTree, ignoreHTTPSErrors, screenshotTaskQueue);
+
     await Promise.all([
       client.send('Network.enable', {}),
-      client.send('Page.enable', {}),
       client.send('Runtime.enable', {}),
       client.send('Security.enable', {}),
       client.send('Performance.enable', {})
     ]);
     if (ignoreHTTPSErrors)
       await client.send('Security.setOverrideCertificateErrors', {override: true});
-    const page = new Page(client, ignoreHTTPSErrors, screenshotTaskQueue);
-    await page.goto('about:blank');
     // Initialize default page size.
     if (!appMode)
       await page.setViewport({width: 800, height: 600});
+
     return page;
   }
 
   /**
    * @param {!Puppeteer.Session} client
+   * @param {{frame: Object, childFrames: ?Array}} frameTree
    * @param {boolean} ignoreHTTPSErrors
    * @param {!Puppeteer.TaskQueue} screenshotTaskQueue
    */
-  constructor(client, ignoreHTTPSErrors, screenshotTaskQueue) {
+  constructor(client, frameTree, ignoreHTTPSErrors, screenshotTaskQueue) {
     super();
     this._client = client;
     this._keyboard = new Keyboard(client);
     this._mouse = new Mouse(client, this._keyboard);
     this._touchscreen = new Touchscreen(client, this._keyboard);
-    this._frameManager = new FrameManager(client, this);
+    this._frameManager = new FrameManager(client, frameTree, this);
     this._networkManager = new NetworkManager(client);
     this._emulationManager = new EmulationManager(client);
     this._tracing = new Tracing(client);

test/golden/reconnect-nested-frames.txt

@@ -0,0 +1,5 @@
+http://localhost:8907/frames/nested-frames.html
+    http://localhost:8907/frames/two-frames.html
+        http://localhost:8907/frames/frame.html
+        http://localhost:8907/frames/frame.html
+    http://localhost:8907/frames/frame.html

test/test.js

@@ -74,6 +74,12 @@ beforeAll(SX(async function() {
     rm(OUTPUT_DIR);
 }));
 
+beforeEach(SX(async function() {
+  server.reset();
+  httpsServer.reset();
+  GoldenUtils.addMatchers(jasmine, GOLDEN_DIR, OUTPUT_DIR);
+}));
+
 afterAll(SX(async function() {
   await Promise.all([
     server.stop(),
@@ -179,11 +185,16 @@ describe('Puppeteer', function() {
     it('should be able to reconnect to a disconnected browser', SX(async function() {
       const originalBrowser = await puppeteer.launch(defaultBrowserOptions);
       const browserWSEndpoint = originalBrowser.wsEndpoint();
+      const page = await originalBrowser.newPage();
+      await page.goto(PREFIX + '/frames/nested-frames.html');
       originalBrowser.disconnect();
 
+      const FrameUtils = require('./frame-utils');
       const browser = await puppeteer.connect({browserWSEndpoint});
-      const page = await browser.newPage();
+      const pages = await browser.pages();
-      expect(await page.evaluate(() => 7 * 8)).toBe(56);
+      const restoredPage = pages.find(page => page.url() === PREFIX + '/frames/nested-frames.html');
+      expect(FrameUtils.dumpFrames(restoredPage.mainFrame())).toBeGolden('reconnect-nested-frames.txt');
+      expect(await restoredPage.evaluate(() => 7 * 8)).toBe(56);
       await browser.close();
     }));
   });
@@ -212,9 +223,6 @@ describe('Page', function() {
 
   beforeEach(SX(async function() {
     page = await browser.newPage();
-    server.reset();
-    httpsServer.reset();
-    GoldenUtils.addMatchers(jasmine, GOLDEN_DIR, OUTPUT_DIR);
   }));
 
   afterEach(SX(async function() {
@@ -2739,6 +2747,91 @@ describe('Page', function() {
 
     }));
   });
+
+  describe('Target', function() {
+    it('Browser.targets should return all of the targets', SX(async function() {
+      // The pages will be the testing page and the original newtab page
+      const targets = browser.targets();
+      expect(targets.some(target => target.type() === 'page' &&
+        target.url() === 'about:blank')).toBeTruthy('Missing blank page');
+      expect(targets.some(target => target.type() === 'other' &&
+        target.url() === '')).toBeTruthy('Missing browser target');
+    }));
+    it('Browser.pages should return all of the pages', SX(async function() {
+      // The pages will be the testing page and the original newtab page
+      const allPages = await browser.pages();
+      expect(allPages.length).toBe(2);
+      expect(allPages).toContain(page);
+      expect(allPages[0]).not.toBe(allPages[1]);
+    }));
+    it('should be able to use the default page in the browser', SX(async function() {
+      // The pages will be the testing page and the original newtab page
+      const allPages = await browser.pages();
+      const originalPage = allPages.find(p => p !== page);
+      expect(await originalPage.evaluate(() => ['Hello', 'world'].join(' '))).toBe('Hello world');
+      expect(await originalPage.$('body')).toBeTruthy();
+    }));
+    it('should report when a new page is created and closed', SX(async function(){
+      const otherPagePromise = new Promise(fulfill => browser.once('targetcreated', target => fulfill(target.page())));
+      await page.evaluate(url => window.open(url), CROSS_PROCESS_PREFIX);
+      const otherPage = await otherPagePromise;
+      expect(otherPage.url()).toContain(CROSS_PROCESS_PREFIX);
+
+      expect(await otherPage.evaluate(() => ['Hello', 'world'].join(' '))).toBe('Hello world');
+      expect(await otherPage.$('body')).toBeTruthy();
+
+      let allPages = await browser.pages();
+      expect(allPages).toContain(page);
+      expect(allPages).toContain(otherPage);
+
+      const closePagePromise = new Promise(fulfill => browser.once('targetdestroyed', target => fulfill(target.page())));
+      await otherPage.close();
+      expect(await closePagePromise).toBe(otherPage);
+
+      allPages = await Promise.all(browser.targets().map(target => target.page()));
+      expect(allPages).toContain(page);
+      expect(allPages).not.toContain(otherPage);
+    }));
+    it('should report when a service worker is created and destroyed', SX(async function() {
+      await page.goto(EMPTY_PAGE);
+      const createdTarget = new Promise(fulfill => browser.once('targetcreated', target => fulfill(target)));
+      const registration = await page.evaluateHandle(() => navigator.serviceWorker.register('sw.js'));
+
+      expect((await createdTarget).type()).toBe('service_worker');
+      expect((await createdTarget).url()).toBe(PREFIX + '/sw.js');
+
+      const destroyedTarget = new Promise(fulfill => browser.once('targetdestroyed', target => fulfill(target)));
+      await page.evaluate(registration => registration.unregister(), registration);
+      expect(await destroyedTarget).toBe(await createdTarget);
+    }));
+    it('should report when a target url changes', SX(async function(){
+      await page.goto(EMPTY_PAGE);
+      let changedTarget = new Promise(fulfill => browser.once('targetchanged', target => fulfill(target)));
+      await page.goto(CROSS_PROCESS_PREFIX + '/');
+      expect((await changedTarget).url()).toBe(CROSS_PROCESS_PREFIX + '/');
+
+      changedTarget = new Promise(fulfill => browser.once('targetchanged', target => fulfill(target)));
+      await page.goto(EMPTY_PAGE);
+      expect((await changedTarget).url()).toBe(EMPTY_PAGE);
+    }));
+    it('should not report uninitialized pages', SX(async function() {
+      browser.on('targetchanged', () => {
+        expect(false).toBe(true, 'target should not be reported as changed');
+      });
+      const targetPromise = new Promise(fulfill => browser.once('targetcreated', target => fulfill(target)));
+      const newPagePromise = browser.newPage();
+      const target = await targetPromise;
+      expect(target.url()).toBe('about:blank');
+
+      const newPage = await newPagePromise;
+      const targetPromise2 = new Promise(fulfill => browser.once('targetcreated', target => fulfill(target)));
+      const evaluatePromise = newPage.evaluate(() => window.open('about:blank'));
+      const target2 = await targetPromise2;
+      expect(target2.url()).toBe('about:blank');
+      await evaluatePromise;
+      await newPage.close();
+    }));
+  });
 });
 
 if (process.env.COVERAGE) {

utils/doclint/check_public_api/index.js

@@ -35,6 +35,7 @@ const EXCLUDE_CLASSES = new Set([
 ]);
 
 const EXCLUDE_METHODS = new Set([
+  'Browser.create',
   'Headers.fromPayload',
   'Page.create',
   'JSHandle.toString',