feat: support Response.buffer(), Response.json() and Response.text() (#4063)

This patch:
- implements Response.buffer() and other methods
- splits out relevant tests into a separate test suites
- implements `testServer.enableGzip()` method to optionally gzip
  certain routes in tests
- adds tests to make sure `Response.text()` returns expected results
  for binary and compressed responses.

experimental/puppeteer-firefox/lib/NetworkManager.js

@@ -60,7 +60,7 @@ class NetworkManager extends EventEmitter {
     const request = this._requests.get(event.requestId);
     if (!request)
       return;
-    const response = new Response(request, event);
+    const response = new Response(this._session, request, event);
     request._response = response;
     this.emit(Events.NetworkManager.Response, response);
   }
@@ -71,8 +71,12 @@ class NetworkManager extends EventEmitter {
       return;
     // Keep redirected requests in the map for future reference in redirectChain.
     const isRedirected = request.response().status() >= 300 && request.response().status() <= 399;
-    if (!isRedirected)
+    if (isRedirected) {
+      request.response()._bodyLoadedPromiseFulfill.call(null, new Error('Response body is unavailable for redirect responses'));
+    } else {
       this._requests.delete(request._id);
+      request.response()._bodyLoadedPromiseFulfill.call(null);
+    }
     this.emit(Events.NetworkManager.RequestFinished, request);
   }
 
@@ -81,6 +85,8 @@ class NetworkManager extends EventEmitter {
     if (!request)
       return;
     this._requests.delete(request._id);
+    if (request.response())
+      request.response()._bodyLoadedPromiseFulfill.call(null);
     request._errorText = event.errorCode;
     this.emit(Events.NetworkManager.RequestFailed, request);
   }
@@ -200,7 +206,8 @@ class Request {
 }
 
 class Response {
-  constructor(request, payload) {
+  constructor(session, request, payload) {
+    this._session = session;
     this._request = request;
     this._remoteIPAddress = payload.remoteIPAddress;
     this._remotePort = payload.remotePort;
@@ -210,6 +217,44 @@ class Response {
     this._securityDetails = payload.securityDetails ? new SecurityDetails(payload.securityDetails) : null;
     for (const {name, value} of payload.headers)
       this._headers[name.toLowerCase()] = value;
+    this._bodyLoadedPromise = new Promise(fulfill => {
+      this._bodyLoadedPromiseFulfill = fulfill;
+    });
+  }
+
+  /**
+   * @return {!Promise<!Buffer>}
+   */
+  buffer() {
+    if (!this._contentPromise) {
+      this._contentPromise = this._bodyLoadedPromise.then(async error => {
+        if (error)
+          throw error;
+        const response = await this._session.send('Network.getResponseBody', {
+          requestId: this._request._id
+        });
+        if (response.evicted)
+          throw new Error(`Response body for ${this._request.method()} ${this._request.url()} was evicted!`);
+        return Buffer.from(response.base64body, 'base64');
+      });
+    }
+    return this._contentPromise;
+  }
+
+  /**
+   * @return {!Promise<string>}
+   */
+  async text() {
+    const content = await this.buffer();
+    return content.toString('utf8');
+  }
+
+  /**
+   * @return {!Promise<!Object>}
+   */
+  async json() {
+    const content = await this.text();
+    return JSON.parse(content);
   }
 
   securityDetails() {

experimental/puppeteer-firefox/misc/puppeteer.cfg

@@ -17,6 +17,12 @@ pref("browser.newtabpage.enabled", false);
 // Disable topstories
 pref("browser.newtabpage.activity-stream.feeds.section.topstories", false);
 
+// DevTools JSONViewer sometimes fails to load dependencies with its require.js.
+// This doesn't affect Puppeteer operations, but spams console with a lot of
+// unpleasant errors.
+// (bug 1424372)
+pref("devtools.jsonview.enabled", false);
+
 // Increase the APZ content response timeout in tests to 1 minute.
 // This is to accommodate the fact that test environments tends to be
 // slower than production environments (with the b2g emulator being

experimental/puppeteer-firefox/package.json

@@ -9,7 +9,7 @@
     "node": ">=8.9.4"
   },
   "puppeteer": {
-    "firefox_revision": "f7b25713dd00f0deda7032dc25a72d4c7b42446e"
+    "firefox_revision": "3ba79216e3c5ae4e85006047cdd93eac4197427d"
   },
   "scripts": {
     "install": "node install.js",

test/navigation.spec.js

@@ -499,7 +499,7 @@ module.exports.addTests = function({testRunner, expect, Errors, CHROME}) {
       const error = await navigationPromise;
       expect(error.message).toBe('Navigating frame was detached');
     });
-    it_fails_ffox('should return matching responses', async({page, server}) => {
+    it('should return matching responses', async({page, server}) => {
       // Disable cache: otherwise, chromium will cache similar requests.
       await page.setCacheEnabled(false);
       await page.goto(server.EMPTY_PAGE);

test/network.spec.js

@@ -156,6 +156,86 @@ module.exports.addTests = function({testRunner, expect, CHROME}) {
     });
   });
 
+  describe('Response.text', function() {
+    it('should work', async({page, server}) => {
+      const response = await page.goto(server.PREFIX + '/simple.json');
+      expect(await response.text()).toBe('{"foo": "bar"}\n');
+    });
+    it('should return uncompressed text', async({page, server}) => {
+      server.enableGzip('/simple.json');
+      const response = await page.goto(server.PREFIX + '/simple.json');
+      expect(response.headers()['content-encoding']).toBe('gzip');
+      expect(await response.text()).toBe('{"foo": "bar"}\n');
+    });
+    it('should throw when requesting body of redirected response', async({page, server}) => {
+      server.setRedirect('/foo.html', '/empty.html');
+      const response = await page.goto(server.PREFIX + '/foo.html');
+      const redirectChain = response.request().redirectChain();
+      expect(redirectChain.length).toBe(1);
+      const redirected = redirectChain[0].response();
+      expect(redirected.status()).toBe(302);
+      let error = null;
+      await redirected.text().catch(e => error = e);
+      expect(error.message).toContain('Response body is unavailable for redirect responses');
+    });
+    it('should wait until response completes', async({page, server}) => {
+      await page.goto(server.EMPTY_PAGE);
+      // Setup server to trap request.
+      let serverResponse = null;
+      server.setRoute('/get', (req, res) => {
+        serverResponse = res;
+        // In Firefox, |fetch| will be hanging until it receives |Content-Type| header
+        // from server.
+        res.setHeader('Content-Type', 'text/plain; charset=utf-8');
+        res.write('hello ');
+      });
+      // Setup page to trap response.
+      let requestFinished = false;
+      page.on('requestfinished', r => requestFinished = requestFinished || r.url().includes('/get'));
+      // send request and wait for server response
+      const [pageResponse] = await Promise.all([
+        page.waitForResponse(r => !utils.isFavicon(r.request())),
+        page.evaluate(() => fetch('./get', { method: 'GET'})),
+        server.waitForRequest('/get'),
+      ]);
+
+      expect(serverResponse).toBeTruthy();
+      expect(pageResponse).toBeTruthy();
+      expect(pageResponse.status()).toBe(200);
+      expect(requestFinished).toBe(false);
+
+      const responseText = pageResponse.text();
+      // Write part of the response and wait for it to be flushed.
+      await new Promise(x => serverResponse.write('wor', x));
+      // Finish response.
+      await new Promise(x => serverResponse.end('ld!', x));
+      expect(await responseText).toBe('hello world!');
+    });
+  });
+
+  describe('Response.json', function() {
+    it('should work', async({page, server}) => {
+      const response = await page.goto(server.PREFIX + '/simple.json');
+      expect(await response.json()).toEqual({foo: 'bar'});
+    });
+  });
+
+  describe('Response.buffer', function() {
+    it('should work', async({page, server}) => {
+      const response = await page.goto(server.PREFIX + '/pptr.png');
+      const imageBuffer = fs.readFileSync(path.join(__dirname, 'assets', 'pptr.png'));
+      const responseBuffer = await response.buffer();
+      expect(responseBuffer.equals(imageBuffer)).toBe(true);
+    });
+    it('should work with compression', async({page, server}) => {
+      server.enableGzip('/pptr.png');
+      const response = await page.goto(server.PREFIX + '/pptr.png');
+      const imageBuffer = fs.readFileSync(path.join(__dirname, 'assets', 'pptr.png'));
+      const responseBuffer = await response.buffer();
+      expect(responseBuffer.equals(imageBuffer)).toBe(true);
+    });
+  });
+
   describe('Network Events', function() {
     it('Page.Events.Request', async({page, server}) => {
       const requests = [];
@@ -193,56 +273,6 @@ module.exports.addTests = function({testRunner, expect, CHROME}) {
       expect(response.statusText()).toBe('cool!');
     });
 
-    it_fails_ffox('Page.Events.Response should provide body', async({page, server}) => {
-      let response = null;
-      page.on('response', r => response = r);
-      await page.goto(server.PREFIX + '/simple.json');
-      expect(response).toBeTruthy();
-      expect(await response.text()).toBe('{"foo": "bar"}\n');
-      expect(await response.json()).toEqual({foo: 'bar'});
-    });
-    it_fails_ffox('Page.Events.Response should throw when requesting body of redirected response', async({page, server}) => {
-      server.setRedirect('/foo.html', '/empty.html');
-      const response = await page.goto(server.PREFIX + '/foo.html');
-      const redirectChain = response.request().redirectChain();
-      expect(redirectChain.length).toBe(1);
-      const redirected = redirectChain[0].response();
-      expect(redirected.status()).toBe(302);
-      let error = null;
-      await redirected.text().catch(e => error = e);
-      expect(error.message).toContain('Response body is unavailable for redirect responses');
-    });
-    it_fails_ffox('Page.Events.Response should not report body unless request is finished', async({page, server}) => {
-      await page.goto(server.EMPTY_PAGE);
-      // Setup server to trap request.
-      let serverResponse = null;
-      server.setRoute('/get', (req, res) => {
-        serverResponse = res;
-        res.write('hello ');
-      });
-      // Setup page to trap response.
-      let pageResponse = null;
-      let requestFinished = false;
-      page.on('response', r => pageResponse = r);
-      page.on('requestfinished', () => requestFinished = true);
-      // send request and wait for server response
-      await Promise.all([
-        page.evaluate(() => fetch('./get', { method: 'GET'})),
-        utils.waitEvent(page, 'response')
-      ]);
-
-      expect(serverResponse).toBeTruthy();
-      expect(pageResponse).toBeTruthy();
-      expect(pageResponse.status()).toBe(200);
-      expect(requestFinished).toBe(false);
-
-      const responseText = pageResponse.text();
-      // Write part of the response and wait for it to be flushed.
-      await new Promise(x => serverResponse.write('wor', x));
-      // Finish response.
-      await new Promise(x => serverResponse.end('ld!', x));
-      expect(await responseText).toBe('hello world!');
-    });
     it('Page.Events.RequestFailed', async({page, server}) => {
       await page.setRequestInterception(true);
       page.on('request', request => {

utils/testserver/index.js

@@ -80,6 +80,8 @@ class TestServer {
     this._auths = new Map();
     /** @type {!Map<string, string>} */
     this._csp = new Map();
+    /** @type {!Set<string>} */
+    this._gzipRoutes = new Set();
     /** @type {!Map<string, !Promise>} */
     this._requestSubscribers = new Map();
   }
@@ -111,6 +113,10 @@ class TestServer {
     this._auths.set(path, {username, password});
   }
 
+  enableGzip(path) {
+    this._gzipRoutes.add(path);
+  }
+
   /**
    * @param {string} path
    * @param {string} csp
@@ -169,6 +175,7 @@ class TestServer {
     this._routes.clear();
     this._auths.clear();
     this._csp.clear();
+    this._gzipRoutes.clear();
     const error = new Error('Static Server has been reset');
     for (const subscriber of this._requestSubscribers.values())
       subscriber[rejectSymbol].call(null, error);
@@ -230,14 +237,22 @@ class TestServer {
     if (this._csp.has(pathName))
       response.setHeader('Content-Security-Policy', this._csp.get(pathName));
 
-    fs.readFile(filePath, function(err, data) {
+    fs.readFile(filePath, (err, data) => {
       if (err) {
         response.statusCode = 404;
         response.end(`File not found: ${filePath}`);
         return;
       }
       response.setHeader('Content-Type', mime.getType(filePath));
-      response.end(data);
+      if (this._gzipRoutes.has(pathName)) {
+        response.setHeader('Content-Encoding', 'gzip');
+        const zlib = require('zlib');
+        zlib.gzip(data, (_, result) => {
+          response.end(result);
+        });
+      } else {
+        response.end(data);
+      }
     });
   }