fix(Navigation): do not race with security error for navigation (#1237)

Currently, NavigationWatcher listens to lifecycle events from Page
domain and security events from Security domain.

However, the events are dispatched from different processes in browser:
- Page's lifecycle events are dispatched from renderer process
- Security events are dispatched from browser process

This makes for the undefined order between events and results in
NavigationWatcher reporting different failuer messages, based on
the event order.

This patch stops relying on security errors in navigation watcher and
instead switches to request failure codes for the main resource.

Fixes #1195

lib/FrameManager.js

@@ -173,10 +173,10 @@ class FrameManager extends EventEmitter {
   }
 
   /**
-   * @return {boolean}
+   * @return {?string}
    */
-  isMainFrameLoadingFailed() {
-    return !!this._mainFrame._loadingFailed;
+  unreachableMainFrameURL() {
+    return this._mainFrame._unreachableURL || null;
   }
 }
 
@@ -477,7 +477,7 @@ class Frame {
   _navigated(framePayload) {
     this._name = framePayload.name;
     this._url = framePayload.url;
-    this._loadingFailed = !!framePayload.unreachableUrl;
+    this._unreachableURL = framePayload.unreachableUrl;
   }
 
   _detach() {

lib/NavigatorWatcher.js

@@ -20,16 +20,14 @@ class NavigatorWatcher {
   /**
    * @param {!Puppeteer.Session} client
    * @param {string} frameId
-   * @param {boolean} ignoreHTTPSErrors
    * @param {!Object=} options
    */
-  constructor(client, frameId, ignoreHTTPSErrors, options = {}) {
+  constructor(client, frameId, options = {}) {
     console.assert(options.networkIdleTimeout === undefined, 'ERROR: networkIdleTimeout option is no longer supported.');
     console.assert(options.networkIdleInflight === undefined, 'ERROR: networkIdleInflight option is no longer supported.');
     console.assert(options.waitUntil !== 'networkidle', 'ERROR: "networkidle" option is no longer supported. Use "networkidle2" instead');
     this._client = client;
     this._frameId = frameId;
-    this._ignoreHTTPSErrors = ignoreHTTPSErrors;
     this._timeout = typeof options.timeout === 'number' ? options.timeout : 30000;
     let waitUntil = ['load'];
     if (Array.isArray(options.waitUntil))
@@ -56,13 +54,6 @@ class NavigatorWatcher {
       navigationPromises.push(watchdog);
     }
 
-    if (!this._ignoreHTTPSErrors) {
-      const certificateError = new Promise(fulfill => {
-        this._eventListeners.push(helper.addEventListener(this._client, 'Security.certificateError', fulfill));
-      }).then(error => 'SSL Certificate error: ' + error.errorType);
-      navigationPromises.push(certificateError);
-    }
-
     this._eventListeners.push(helper.addEventListener(this._client, 'Page.lifecycleEvent', this._onLifecycleEvent.bind(this)));
     const pendingEventsFired = new Promise(fulfill => this._pendingEventsCallback = fulfill);
     navigationPromises.push(pendingEventsFired);

lib/Page.js

@@ -457,9 +457,9 @@ class Page extends EventEmitter {
    */
   async goto(url, options) {
     const mainFrame = this._frameManager.mainFrame();
-    const watcher = new NavigatorWatcher(this._client, mainFrame._id, this._ignoreHTTPSErrors, options);
-    const responses = new Map();
-    const listener = helper.addEventListener(this._networkManager, NetworkManager.Events.Response, response => responses.set(response.url, response));
+    const watcher = new NavigatorWatcher(this._client, mainFrame._id, options);
+    const requests = new Map();
+    const listener = helper.addEventListener(this._networkManager, NetworkManager.Events.Request, request => requests.set(request.url, request));
     const navigationPromise = watcher.waitForNavigation();
 
     const referrer = this._networkManager.extraHTTPHeaders()['referer'];
@@ -475,9 +475,14 @@ class Page extends EventEmitter {
     helper.removeEventListeners([listener]);
     if (error)
       throw error;
-    if (this._frameManager.isMainFrameLoadingFailed())
-      throw new Error('Failed to navigate: ' + url);
-    return responses.get(this.mainFrame().url()) || null;
+    const unreachableURL = this._frameManager.unreachableMainFrameURL();
+    if (unreachableURL) {
+      const request = requests.get(unreachableURL) || null;
+      const failure = request ? request.failure() : null;
+      throw new Error('Failed to navigate: ' + url + (failure ? ' ' + failure.errorText : ''));
+    }
+    const request = requests.get(this.mainFrame().url());
+    return request ? request.response() : null;
   }
 
   /**
@@ -498,7 +503,7 @@ class Page extends EventEmitter {
    */
   async waitForNavigation(options) {
     const mainFrame = this._frameManager.mainFrame();
-    const watcher = new NavigatorWatcher(this._client, mainFrame._id, this._ignoreHTTPSErrors, options);
+    const watcher = new NavigatorWatcher(this._client, mainFrame._id, options);
 
     const responses = new Map();
     const listener = helper.addEventListener(this._networkManager, NetworkManager.Events.Response, response => responses.set(response.url, response));

test/test.js

@@ -89,7 +89,7 @@ afterAll(SX(async function() {
 
 describe('Puppeteer', function() {
   describe('Puppeteer.launch', function() {
-    xit('should support ignoreHTTPSErrors option', SX(async function() {
+    it('should support ignoreHTTPSErrors option', SX(async function() {
       const options = Object.assign({ignoreHTTPSErrors: true}, defaultBrowserOptions);
       const browser = await puppeteer.launch(options);
       const page = await browser.newPage();
@@ -909,7 +909,14 @@ describe('Page', function() {
       page.on('requestfailed', request => expect(request).toBeTruthy());
       let error = null;
       await page.goto(HTTPS_PREFIX + '/empty.html').catch(e => error = e);
-      expect(error.message).toContain('SSL Certificate error');
+      expect(error.message).toContain('net::ERR_INSECURE_RESPONSE');
+    }));
+    it('should fail when navigating to bad SSL after redirects', SX(async function() {
+      server.setRedirect('/redirect/1.html', '/redirect/2.html');
+      server.setRedirect('/redirect/2.html', '/empty.html');
+      let error = null;
+      await page.goto(HTTPS_PREFIX + '/redirect/1.html').catch(e => error = e);
+      expect(error.message).toContain('net::ERR_INSECURE_RESPONSE');
     }));
     it('should throw if networkidle is passed as an option', SX(async function() {
       let error = null;