Implement browser 'ignoreHTTPSErrors' option. (#177)

This patch implements Browser 'ignoreHTTPSErrors' option.

Fixes #84.

docs/api.md

@@ -181,8 +181,9 @@ browser.newPage().then(async page => {
 
 #### new Browser([options])
 - `options` <[Object]>  Set of configurable options to set on the browser. Can have the following fields:
-	- `headless` <[boolean]> Whether to run chromium in [headless mode](https://developers.google.com/web/updates/2017/04/headless-chrome). Defaults to `true`.
-	- `executablePath` <[string]> Path to a chromium executable to run instead of bundled chromium. If `executablePath` is a relative path, then it is resolved relative to [current working directory](https://nodejs.org/api/process.html#process_process_cwd).
+  - `ignoreHTTPSErrors` <[boolean]> Whether to ignore HTTPS errors during navigation. Defaults to `false`.
+  - `headless` <[boolean]> Whether to run chromium in [headless mode](https://developers.google.com/web/updates/2017/04/headless-chrome). Defaults to `true`.
+  - `executablePath` <[string]> Path to a chromium executable to run instead of bundled chromium. If `executablePath` is a relative path, then it is resolved relative to [current working directory](https://nodejs.org/api/process.html#process_process_cwd).
   - `slowMo` <[number]> Slows down Puppeteer operations by the specified amount of milliseconds. Useful
 so that you can see what is going on.
   - `args` <[Array]<[string]>> Additional arguments to pass to the chromium instance. List of chromium flags can be found [here](http://peter.sh/experiments/chromium-command-line-switches/).
@@ -566,6 +567,7 @@ The extra HTTP headers will be sent with every request the page initiates.
 
 > **NOTE** page.setExtraHTTPHeaders does not guarantee the order of headers in the outgoing requests.
 
+
 #### page.setInPageCallback(name, callback)
 - `name` <[string]> Name of the callback to be assigned on window object
 - `callback` <[function]> Callback function which will be called in puppeteer's context.

lib/Browser.js

@@ -72,6 +72,7 @@ class Browser {
       console.assert(revisionInfo, 'Chromium revision is not downloaded. Run npm install');
       this._chromeExecutable = revisionInfo.executablePath;
     }
+    this._ignoreHTTPSErrors = !!options.ignoreHTTPSErrors;
     if (Array.isArray(options.args))
       this._chromeArguments.push(...options.args);
     this._connectionDelay = options.slowMo || 0;
@@ -92,7 +93,7 @@ class Browser {
     if (!this._chromeProcess || this._terminated)
       throw new Error('ERROR: this chrome instance is not alive any more!');
     let client = await Connection.create(this._remoteDebuggingPort, this._connectionDelay);
-    let page = await Page.create(client, this._screenshotTaskQueue);
+    let page = await Page.create(client, this._ignoreHTTPSErrors, this._screenshotTaskQueue);
     return page;
   }
 

lib/NavigatorWatcher.js

@@ -19,10 +19,12 @@ const helper = require('./helper');
 class NavigatorWatcher {
   /**
    * @param {!Connection} client
+   * @param {boolean} ignoreHTTPSErrors
    * @param {!Object=} options
    */
-  constructor(client, options = {}) {
+  constructor(client, ignoreHTTPSErrors, options = {}) {
     this._client = client;
+    this._ignoreHTTPSErrors = ignoreHTTPSErrors;
     this._timeout = typeof options['timeout'] === 'number' ? options['timeout'] : 30000;
     this._idleTime = typeof options['networkIdleTimeout'] === 'number' ? options['networkIdleTimeout'] : 1000;
     this._idleInflight = typeof options['networkIdleInflight'] === 'number' ? options['networkIdleInflight'] : 2;
@@ -30,7 +32,6 @@ class NavigatorWatcher {
     console.assert(this._waitUntil === 'load' || this._waitUntil === 'networkidle', 'Unknown value for options.waitUntil: ' + this._waitUntil);
   }
 
-
   /**
    * @return {!Promise<!Map<string, !Response>>}
    */
@@ -38,34 +39,40 @@ class NavigatorWatcher {
     this._inflightRequests = 0;
     this._requestIds = new Set();
 
-    this._eventListeners = [
-      helper.addEventListener(this._client, 'Network.requestWillBeSent', this._onLoadingStarted.bind(this)),
-      helper.addEventListener(this._client, 'Network.loadingFinished', this._onLoadingCompleted.bind(this)),
-      helper.addEventListener(this._client, 'Network.loadingFailed', this._onLoadingCompleted.bind(this)),
-      helper.addEventListener(this._client, 'Network.webSocketCreated', this._onLoadingStarted.bind(this)),
-      helper.addEventListener(this._client, 'Network.webSocketClosed', this._onLoadingCompleted.bind(this)),
-    ];
-
-    let certificateError = new Promise(fulfill => {
-      this._eventListeners.push(helper.addEventListener(this._client, 'Security.certificateError', fulfill));
-    }).then(error => 'SSL Certificate error: ' + error.errorType);
-
-    let networkIdle = new Promise(fulfill => this._networkIdleCallback = fulfill).then(() => null);
-    let loadEventFired = new Promise(fulfill => {
-      this._eventListeners.push(helper.addEventListener(this._client, 'Page.loadEventFired', fulfill));
-    }).then(() => null);
+    this._eventListeners = [];
 
     let watchdog = new Promise(fulfill => this._maximumTimer = setTimeout(fulfill, this._timeout))
         .then(() => 'Navigation Timeout Exceeded: ' + this._timeout + 'ms exceeded');
+    let navigationPromises = [watchdog];
 
-    try {
-      // Await for the command to throw exception in case of illegal arguments.
-      const error = await Promise.race([certificateError, watchdog, this._waitUntil === 'load' ? loadEventFired : networkIdle]);
-      if (error)
-        throw new Error(error);
-    } finally {
-      this._cleanup();
+    if (!this._ignoreHTTPSErrors) {
+      let certificateError = new Promise(fulfill => {
+        this._eventListeners.push(helper.addEventListener(this._client, 'Security.certificateError', fulfill));
+      }).then(error => 'SSL Certificate error: ' + error.errorType);
+      navigationPromises.push(certificateError);
     }
+
+    if (this._waitUntil === 'load') {
+      let loadEventFired = new Promise(fulfill => {
+        this._eventListeners.push(helper.addEventListener(this._client, 'Page.loadEventFired', fulfill));
+      }).then(() => null);
+      navigationPromises.push(loadEventFired);
+    } else {
+      this._eventListeners.push(...[
+        helper.addEventListener(this._client, 'Network.requestWillBeSent', this._onLoadingStarted.bind(this)),
+        helper.addEventListener(this._client, 'Network.loadingFinished', this._onLoadingCompleted.bind(this)),
+        helper.addEventListener(this._client, 'Network.loadingFailed', this._onLoadingCompleted.bind(this)),
+        helper.addEventListener(this._client, 'Network.webSocketCreated', this._onLoadingStarted.bind(this)),
+        helper.addEventListener(this._client, 'Network.webSocketClosed', this._onLoadingCompleted.bind(this)),
+      ]);
+      let networkIdle = new Promise(fulfill => this._networkIdleCallback = fulfill).then(() => null);
+      navigationPromises.push(networkIdle);
+    }
+
+    const error = await Promise.race(navigationPromises);
+    this._cleanup();
+    if (error)
+      throw new Error(error);
   }
 
   cancel() {
@@ -97,9 +104,6 @@ class NavigatorWatcher {
       this._idleTimer = setTimeout(this._networkIdleCallback, this._idleTime);
   }
 
-  _init() {
-  }
-
   _cleanup() {
     helper.removeEventListeners(this._eventListeners);
 

lib/Page.js

@@ -28,17 +28,20 @@ let helper = require('./helper');
 class Page extends EventEmitter {
   /**
    * @param {!Connection} client
+   * @param {boolean} ignoreHTTPSErrors
    * @param {!TaskQueue} screenshotTaskQueue
    * @return {!Promise<!Page>}
    */
-  static async create(client, screenshotTaskQueue) {
+  static async create(client, ignoreHTTPSErrors, screenshotTaskQueue) {
     await Promise.all([
       client.send('Network.enable', {}),
       client.send('Page.enable', {}),
       client.send('Runtime.enable', {}),
       client.send('Security.enable', {}),
     ]);
-    const page = new Page(client, screenshotTaskQueue);
+    if (ignoreHTTPSErrors)
+      await client.send('Security.setOverrideCertificateErrors', {override: true});
+    const page = new Page(client, ignoreHTTPSErrors, screenshotTaskQueue);
     await page.navigate('about:blank');
     // Initialize default page size.
     await page.setViewport({width: 400, height: 300});
@@ -47,9 +50,10 @@ class Page extends EventEmitter {
 
   /**
    * @param {!Connection} client
+   * @param {boolean} ignoreHTTPSErrors
    * @param {!TaskQueue} screenshotTaskQueue
    */
-  constructor(client, screenshotTaskQueue) {
+  constructor(client, ignoreHTTPSErrors, screenshotTaskQueue) {
     super();
     this._client = client;
     this._keyboard = new Keyboard(client);
@@ -59,6 +63,7 @@ class Page extends EventEmitter {
     this._emulationManager = new EmulationManager(client);
     /** @type {!Map<string, function>} */
     this._inPageCallbacks = new Map();
+    this._ignoreHTTPSErrors = ignoreHTTPSErrors;
 
     this._screenshotTaskQueue = screenshotTaskQueue;
 
@@ -76,6 +81,7 @@ class Page extends EventEmitter {
     client.on('Runtime.consoleAPICalled', event => this._onConsoleAPI(event));
     client.on('Page.javascriptDialogOpening', event => this._onDialog(event));
     client.on('Runtime.exceptionThrown', exception => this._handleException(exception.exceptionDetails));
+    client.on('Security.certificateError', event => this._onCertificateError(event));
   }
 
   /**
@@ -106,6 +112,18 @@ class Page extends EventEmitter {
     return this._networkManager.setRequestInterceptor(interceptor);
   }
 
+  /**
+   * @param {!Object} event
+   */
+  _onCertificateError(event) {
+    if (!this._ignoreHTTPSErrors)
+      return;
+    this._client.send('Security.handleCertificateError', {
+      eventId: event.eventId,
+      action: 'continue'
+    });
+  }
+
   /**
    * @param {string} url
    * @return {!Promise}
@@ -238,7 +256,7 @@ class Page extends EventEmitter {
    * @return {!Promise<?Response>}
    */
   async navigate(url, options) {
-    const watcher = new NavigatorWatcher(this._client, options);
+    const watcher = new NavigatorWatcher(this._client, this._ignoreHTTPSErrors, options);
     const responses = new Map();
     const listener = helper.addEventListener(this._networkManager, NetworkManager.Events.Response, response => responses.set(response.url, response));
     const result = watcher.waitForNavigation();
@@ -275,7 +293,7 @@ class Page extends EventEmitter {
    * @return {!Promise<!Response>}
    */
   async waitForNavigation(options) {
-    const watcher = new NavigatorWatcher(this._client, options);
+    const watcher = new NavigatorWatcher(this._client, this._ignoreHTTPSErrors, options);
 
     const responses = new Map();
     const listener = helper.addEventListener(this._networkManager, NetworkManager.Events.Response, response => responses.set(response.url, response));

test/test.js

@@ -52,26 +52,49 @@ else
   console.assert(revisionInfo, `Chromium r${chromiumRevision} is not downloaded. Run 'npm install' and try to re-run tests.`);
 }
 
-describe('Puppeteer', function() {
+let server;
+let httpsServer;
+beforeAll(SX(async function() {
+  const assetsPath = path.join(__dirname, 'assets');
+  server = await SimpleServer.create(assetsPath, PORT);
+  httpsServer = await SimpleServer.createHTTPS(assetsPath, HTTPS_PORT);
+  if (fs.existsSync(OUTPUT_DIR))
+    rm(OUTPUT_DIR);
+}));
+
+afterAll(SX(async function() {
+  await Promise.all([
+    server.stop(),
+    httpsServer.stop(),
+  ]);
+}));
+
+describe('Browser', function() {
+  it('Browser.Options.ignoreHTTPSErrors', SX(async function() {
+    let browser = new Browser({ignoreHTTPSErrors: true, headless, slowMo, args: ['--no-sandbox']});
+    let page = await browser.newPage();
+    let error = null;
+    let response = null;
+    try {
+      response = await page.navigate(HTTPS_PREFIX + '/empty.html');
+    } catch (e) {
+      error = e;
+    }
+    expect(error).toBe(null);
+    expect(response.ok).toBe(true);
+    browser.close();
+  }));
+});
+
+describe('Page', function() {
   let browser;
-  let server;
-  let httpsServer;
   let page;
 
   beforeAll(SX(async function() {
     browser = new Browser({headless, slowMo, args: ['--no-sandbox']});
-    const assetsPath = path.join(__dirname, 'assets');
-    server = await SimpleServer.create(assetsPath, PORT);
-    httpsServer = await SimpleServer.createHTTPS(assetsPath, HTTPS_PORT);
-    if (fs.existsSync(OUTPUT_DIR))
-      rm(OUTPUT_DIR);
   }));
 
   afterAll(SX(async function() {
-    await Promise.all([
-      server.stop(),
-      httpsServer.stop(),
-    ]);
     browser.close();
   }));
 
@@ -459,6 +482,8 @@ describe('Puppeteer', function() {
       expect(error.message).toContain('Cannot navigate to invalid URL');
     }));
     it('should fail when navigating to bad SSL', SX(async function() {
+      // Make sure that network events do not emit 'undefind'.
+      // @see https://github.com/GoogleChrome/puppeteer/issues/168
       page.on('request', request => expect(request).toBeTruthy());
       page.on('requestfinished', request => expect(request).toBeTruthy());
       page.on('requestfailed', request => expect(request).toBeTruthy());