feat: Attribute network requests to frames (#1646)

This patch introduces `request.frame()` method that returns the frame that
initiated request.

Fixes #1579

docs/api.md

@@ -179,6 +179,7 @@
   * [request.abort([errorCode])](#requestaborterrorcode)
   * [request.continue([overrides])](#requestcontinueoverrides)
   * [request.failure()](#requestfailure)
+  * [request.frame()](#requestframe)
   * [request.headers()](#requestheaders)
   * [request.method()](#requestmethod)
   * [request.postData()](#requestpostdata)
@@ -2061,6 +2062,9 @@ page.on('requestfailed', request => {
 });
 ```
 
+#### request.frame()
+- returns: <?[Frame]> A matching [Frame] object, or `null` if navigating to error pages.
+
 #### request.headers()
 - returns: <[Object]> An object with HTTP headers associated with the request. All header names are lower-case.
 

lib/FrameManager.js

@@ -87,6 +87,14 @@ class FrameManager extends EventEmitter {
     return Array.from(this._frames.values());
   }
 
+  /**
+   * @param {!string} frameId
+   * @return {?Frame}
+   */
+  frame(frameId) {
+    return this._frames.get(frameId) || null;
+  }
+
   /**
    * @param {string} frameId
    * @param {?string} parentFrameId

lib/NetworkManager.js

@@ -19,11 +19,13 @@ const Multimap = require('./Multimap');
 
 class NetworkManager extends EventEmitter {
   /**
-   * @param {Puppeteer.Session} client
+   * @param {!Puppeteer.Session} client
+   * @param {!Puppeteer.FrameManager} frameManager
    */
-  constructor(client) {
+  constructor(client, frameManager) {
     super();
     this._client = client;
+    this._frameManager = frameManager;
     /** @type {!Map<string, !Request>} */
     this._requestIdToRequest = new Map();
     /** @type {!Map<string, !Request>} */
@@ -151,17 +153,17 @@ class NetworkManager extends EventEmitter {
       const request = this._interceptionIdToRequest.get(event.interceptionId);
       console.assert(request, 'INTERNAL ERROR: failed to find request for interception redirect.');
       this._handleRequestRedirect(request, event.responseStatusCode, event.responseHeaders);
-      this._handleRequestStart(request._requestId, event.interceptionId, event.redirectUrl, event.resourceType, event.request);
+      this._handleRequestStart(request._requestId, event.interceptionId, event.redirectUrl, event.resourceType, event.request, event.frameId);
       return;
     }
     const requestHash = generateRequestHash(event.request);
     const requestId = this._requestHashToRequestIds.firstValue(requestHash);
     if (requestId) {
       this._requestHashToRequestIds.delete(requestHash, requestId);
-      this._handleRequestStart(requestId, event.interceptionId, event.request.url, event.resourceType, event.request);
+      this._handleRequestStart(requestId, event.interceptionId, event.request.url, event.resourceType, event.request, event.frameId);
     } else {
       this._requestHashToInterceptionIds.set(requestHash, event.interceptionId);
-      this._handleRequestStart(null, event.interceptionId, event.request.url, event.resourceType, event.request);
+      this._handleRequestStart(null, event.interceptionId, event.request.url, event.resourceType, event.request, event.frameId);
     }
   }
 
@@ -186,9 +188,13 @@ class NetworkManager extends EventEmitter {
    * @param {string} url
    * @param {string} resourceType
    * @param {!Object} requestPayload
+   * @param {?string} frameId
    */
-  _handleRequestStart(requestId, interceptionId, url, resourceType, requestPayload) {
+  _handleRequestStart(requestId, interceptionId, url, resourceType, requestPayload, frameId) {
-    const request = new Request(this._client, requestId, interceptionId, this._userRequestInterceptionEnabled, url, resourceType, requestPayload);
+    let frame = null;
+    if (frameId)
+      frame = this._frameManager.frame(frameId);
+    const request = new Request(this._client, requestId, interceptionId, this._userRequestInterceptionEnabled, url, resourceType, requestPayload, frame);
     if (requestId)
       this._requestIdToRequest.set(requestId, request);
     if (interceptionId)
@@ -222,7 +228,7 @@ class NetworkManager extends EventEmitter {
       if (request)
         this._handleRequestRedirect(request, event.redirectResponse.status, event.redirectResponse.headers);
     }
-    this._handleRequestStart(event.requestId, null, event.request.url, event.type, event.request);
+    this._handleRequestStart(event.requestId, null, event.request.url, event.type, event.request, event.frameId);
   }
 
   /**
@@ -281,8 +287,9 @@ class Request {
    * @param {string} url
    * @param {string} resourceType
    * @param {!Object} payload
+   * @param {?Puppeteer.Frame} frame
    */
-  constructor(client, requestId, interceptionId, allowInterception, url, resourceType, payload) {
+  constructor(client, requestId, interceptionId, allowInterception, url, resourceType, payload, frame) {
     this._client = client;
     this._requestId = requestId;
     this._interceptionId = interceptionId;
@@ -299,6 +306,7 @@ class Request {
     this._method = payload.method;
     this._postData = payload.postData;
     this._headers = {};
+    this._frame = frame;
     for (const key of Object.keys(payload.headers))
       this._headers[key.toLowerCase()] = payload.headers[key];
   }
@@ -345,6 +353,13 @@ class Request {
     return this._response;
   }
 
+  /**
+   * @return {?Puppeteer.Frame}
+   */
+  frame() {
+    return this._frame;
+  }
+
   /**
    * @return {?{errorText: string}}
    */

lib/Page.js

@@ -72,7 +72,7 @@ class Page extends EventEmitter {
     this._mouse = new Mouse(client, this._keyboard);
     this._touchscreen = new Touchscreen(client, this._keyboard);
     this._frameManager = new FrameManager(client, frameTree, this);
-    this._networkManager = new NetworkManager(client);
+    this._networkManager = new NetworkManager(client, this._frameManager);
     this._emulationManager = new EmulationManager(client);
     this._tracing = new Tracing(client);
     /** @type {!Map<string, Function>} */

test/test.js

@@ -29,6 +29,7 @@ const iPhoneLandscape = DeviceDescriptors['iPhone 6 landscape'];
 
 const SimpleServer = require('./server/SimpleServer');
 const GoldenUtils = require('./golden-utils');
+const FrameUtils = require('./frame-utils');
 
 const YELLOW_COLOR = '\x1b[33m';
 const RESET_COLOR = '\x1b[0m';
@@ -229,7 +230,6 @@ describe('Puppeteer', function() {
       await page.goto(server.PREFIX + '/frames/nested-frames.html');
       originalBrowser.disconnect();
 
-      const FrameUtils = require('./frame-utils');
       const browser = await puppeteer.connect({browserWSEndpoint});
       const pages = await browser.pages();
       const restoredPage = pages.find(page => page.url() === server.PREFIX + '/frames/nested-frames.html');
@@ -433,7 +433,6 @@ describe('Page', function() {
       expect(error.message).toContain('JSHandle is disposed');
     });
     it('should throw if elementHandles are from other frames', async({page, server}) => {
-      const FrameUtils = require('./frame-utils');
       await FrameUtils.attachFrame(page, 'frame1', server.EMPTY_PAGE);
       const bodyHandle = await page.frames()[1].$('body');
       let error = null;
@@ -601,7 +600,6 @@ describe('Page', function() {
   });
 
   describe('Frame.context', function() {
-    const FrameUtils = require('./frame-utils');
     it('should work', async({page, server}) => {
       await page.goto(server.EMPTY_PAGE);
       await FrameUtils.attachFrame(page, 'frame1', server.EMPTY_PAGE);
@@ -627,7 +625,6 @@ describe('Page', function() {
   });
 
   describe('Frame.evaluate', function() {
-    const FrameUtils = require('./frame-utils');
     it('should have different execution contexts', async({page, server}) => {
       await page.goto(server.EMPTY_PAGE);
       await FrameUtils.attachFrame(page, 'frame1', server.EMPTY_PAGE);
@@ -718,7 +715,6 @@ describe('Page', function() {
   });
 
   describe('Frame.waitForSelector', function() {
-    const FrameUtils = require('./frame-utils');
     const addElement = tag => document.body.appendChild(document.createElement(tag));
 
     it('should immediately resolve promise if node exists', async({page, server}) => {
@@ -1308,6 +1304,8 @@ describe('Page', function() {
         expect(request.method()).toBe('GET');
         expect(request.postData()).toBe(undefined);
         expect(request.resourceType()).toBe('document');
+        expect(request.frame() === page.mainFrame()).toBe(true);
+        expect(request.frame().url()).toBe('about:blank');
         request.continue();
       });
       const response = await page.goto(server.EMPTY_PAGE);
@@ -1621,13 +1619,18 @@ describe('Page', function() {
       const requests = [];
       page.on('request', request => requests.push(request));
       await page.goto(server.EMPTY_PAGE);
-      expect(requests.length).toBe(1);
+      await FrameUtils.attachFrame(page, 'frame1', server.EMPTY_PAGE);
-      expect(requests[0].url()).toContain('empty.html');
+      expect(requests.length).toBe(2);
+      expect(requests[0].url()).toBe(server.EMPTY_PAGE);
+      expect(requests[0].frame() === page.mainFrame()).toBe(true);
+      expect(requests[0].frame().url()).toBe(server.EMPTY_PAGE);
+      expect(requests[1].url()).toBe(server.EMPTY_PAGE);
+      expect(requests[1].frame() === page.frames()[1]).toBe(true);
+      expect(requests[1].frame().url()).toBe(server.EMPTY_PAGE);
     });
   });
 
   describe('Frame Management', function() {
-    const FrameUtils = require('./frame-utils');
     it('should handle nested frames', async({page, server}) => {
       await page.goto(server.PREFIX + '/frames/nested-frames.html');
       expect(FrameUtils.dumpFrames(page.mainFrame())).toBeGolden('nested-frames.txt');
@@ -1646,7 +1649,7 @@ describe('Page', function() {
       page.on('framenavigated', frame => navigatedFrames.push(frame));
       await FrameUtils.navigateFrame(page, 'frame1', './empty.html');
       expect(navigatedFrames.length).toBe(1);
-      expect(navigatedFrames[0].url()).toContain('/empty.html');
+      expect(navigatedFrames[0].url()).toBe(server.EMPTY_PAGE);
 
       // validate framedetached events
       const detachedFrames = [];
@@ -2342,7 +2345,6 @@ describe('Page', function() {
     it('should click the button inside an iframe', async({page, server}) => {
       await page.goto(server.EMPTY_PAGE);
       await page.setContent('<div style="width:100px;height:100px">spacer</div>');
-      const FrameUtils = require('./frame-utils');
       await FrameUtils.attachFrame(page, 'button-test', server.PREFIX + '/input/button.html');
       const frame = page.frames()[1];
       const button = await frame.$('button');
@@ -2353,7 +2355,6 @@ describe('Page', function() {
       await page.setViewport({width: 400, height: 400, deviceScaleFactor: 5});
       expect(await page.evaluate(() => window.devicePixelRatio)).toBe(5);
       await page.setContent('<div style="width:100px;height:100px">spacer</div>');
-      const FrameUtils = require('./frame-utils');
       await FrameUtils.attachFrame(page, 'button-test', server.PREFIX + '/input/button.html');
       const frame = page.frames()[1];
       const button = await frame.$('button');
@@ -2517,6 +2518,8 @@ describe('Page', function() {
       expect(requests[0].resourceType()).toBe('document');
       expect(requests[0].method()).toBe('GET');
       expect(requests[0].response()).toBeTruthy();
+      expect(requests[0].frame() === page.mainFrame()).toBe(true);
+      expect(requests[0].frame().url()).toBe(server.EMPTY_PAGE);
     });
     it('Page.Events.Request should report post data', async({page, server}) => {
       await page.goto(server.EMPTY_PAGE);
@@ -2592,6 +2595,7 @@ describe('Page', function() {
       expect(failedRequests[0].response()).toBe(null);
       expect(failedRequests[0].resourceType()).toBe('stylesheet');
       expect(failedRequests[0].failure().errorText).toBe('net::ERR_FAILED');
+      expect(failedRequests[0].frame()).toBeTruthy();
     });
     it('Page.Events.RequestFinished', async({page, server}) => {
       const requests = [];
@@ -2600,6 +2604,8 @@ describe('Page', function() {
       expect(requests.length).toBe(1);
       expect(requests[0].url()).toBe(server.EMPTY_PAGE);
       expect(requests[0].response()).toBeTruthy();
+      expect(requests[0].frame() === page.mainFrame()).toBe(true);
+      expect(requests[0].frame().url()).toBe(server.EMPTY_PAGE);
     });
     it('should fire events in proper order', async({page, server}) => {
       const events = [];