chore: network module for BiDi (#10159)

packages/puppeteer-core/src/api/HTTPResponse.ts

@@ -67,7 +67,9 @@ export class HTTPResponse {
    * True if the response was successful (status in the range 200-299).
    */
   ok(): boolean {
-    throw new Error('Not implemented');
+    // TODO: document === 0 case?
+    const status = this.status();
+    return status === 0 || (status >= 200 && status <= 299);
   }
 
   /**

packages/puppeteer-core/src/common/Connection.ts

@@ -22,7 +22,7 @@ import {createDeferredPromise} from '../util/util.js';
 
 import {ConnectionTransport} from './ConnectionTransport.js';
 import {debug} from './Debug.js';
-import {ProtocolError} from './Errors.js';
+import {TargetCloseError, ProtocolError} from './Errors.js';
 import {EventEmitter} from './EventEmitter.js';
 
 const debugProtocolSend = debug('puppeteer:protocol:SEND ►');
@@ -150,10 +150,18 @@ export class CallbackRegistry {
     this._reject(callback, message, originalMessage);
   }
 
-  _reject(callback: Callback, message: string, originalMessage?: string): void {
+  _reject(
+    callback: Callback,
+    errorMessage: string | ProtocolError,
+    originalMessage?: string
+  ): void {
+    const isError = errorMessage instanceof ProtocolError;
+    const message = isError ? errorMessage.message : errorMessage;
+    const error = isError ? errorMessage : callback.error;
+
     callback.reject(
       rewriteError(
-        callback.error,
+        error,
         `Protocol error (${callback.label}): ${message}`,
         originalMessage
       )
@@ -171,7 +179,7 @@ export class CallbackRegistry {
   clear(): void {
     for (const callback of this.#callbacks.values()) {
       // TODO: probably we can accept error messages as params.
-      this._reject(callback, 'Target closed');
+      this._reject(callback, new TargetCloseError('Target closed'));
     }
     this.#callbacks.clear();
   }
@@ -513,7 +521,7 @@ export class CDPSessionImpl extends CDPSession {
   ): Promise<ProtocolMapping.Commands[T]['returnType']> {
     if (!this.#connection) {
       return Promise.reject(
-        new Error(
+        new TargetCloseError(
           `Protocol error (${method}): Session closed. Most likely the ${
             this.#targetType
           } has been closed.`
@@ -607,9 +615,6 @@ function rewriteError(
 /**
  * @internal
  */
-export function isTargetClosedError(err: Error): boolean {
+export function isTargetClosedError(error: Error): boolean {
-  return (
+  return error instanceof TargetCloseError;
-    err.message.includes('Target closed') ||
-    err.message.includes('Session closed')
-  );
 }

packages/puppeteer-core/src/common/Errors.ts

@@ -74,6 +74,11 @@ export class ProtocolError extends CustomError {
   }
 }
 
+/**
+ * @internal
+ */
+export class TargetCloseError extends ProtocolError {}
+
 /**
  * @deprecated Do not use.
  *

packages/puppeteer-core/src/common/HTTPResponse.ts

@@ -114,11 +114,6 @@ export class HTTPResponse extends BaseHTTPResponse {
     return this.#url;
   }
 
-  override ok(): boolean {
-    // TODO: document === 0 case?
-    return this.#status === 0 || (this.#status >= 200 && this.#status <= 299);
-  }
-
   override status(): number {
     return this.#status;
   }

packages/puppeteer-core/src/common/Page.ts

@@ -60,6 +60,7 @@ import {Coverage} from './Coverage.js';
 import {DeviceRequestPrompt} from './DeviceRequestPrompt.js';
 import {Dialog} from './Dialog.js';
 import {EmulationManager} from './EmulationManager.js';
+import {TargetCloseError} from './Errors.js';
 import {FileChooser} from './FileChooser.js';
 import {FrameManager, FrameManagerEmittedEvents} from './FrameManager.js';
 import {Keyboard, Mouse, Touchscreen} from './Input.js';
@@ -933,7 +934,7 @@ export class CDPPage extends Page {
     if (!this.#disconnectPromise) {
       this.#disconnectPromise = new Promise(fulfill => {
         return this.#client.once(CDPSessionEmittedEvents.Disconnected, () => {
-          return fulfill(new Error('Target closed'));
+          return fulfill(new TargetCloseError('Target closed'));
         });
       });
     }

packages/puppeteer-core/src/common/bidi/BidiOverCDP.ts

@@ -19,6 +19,7 @@ import * as Bidi from 'chromium-bidi/lib/cjs/protocol/protocol.js';
 import type {ProtocolMapping} from 'devtools-protocol/types/protocol-mapping.js';
 
 import {CDPSession, Connection as CDPPPtrConnection} from '../Connection.js';
+import {TargetCloseError} from '../Errors.js';
 import {Handler} from '../EventEmitter.js';
 
 import {Connection as BidiPPtrConnection} from './Connection.js';
@@ -147,6 +148,10 @@ class CDPClientAdapter<T extends Pick<CDPPPtrConnection, 'send' | 'on' | 'off'>>
     this.#client.off('*', this.#forwardMessage as Handler<any>);
     this.#closed = true;
   }
+
+  isCloseError(error: any): boolean {
+    return error instanceof TargetCloseError;
+  }
 }
 
 /**

packages/puppeteer-core/src/common/bidi/Browser.ts

@@ -33,7 +33,7 @@ import {Connection} from './Connection.js';
  * @internal
  */
 export class Browser extends BrowserBase {
-  static readonly subscribeModules = ['browsingContext'];
+  static readonly subscribeModules = ['browsingContext', 'network'];
 
   static async create(opts: Options): Promise<Browser> {
     // TODO: await until the connection is established.

packages/puppeteer-core/src/common/bidi/BrowsingContext.ts

@@ -0,0 +1,314 @@
+import * as Bidi from 'chromium-bidi/lib/cjs/protocol/protocol.js';
+import ProtocolMapping from 'devtools-protocol/types/protocol-mapping.js';
+
+import {assert} from '../../util/assert.js';
+import {stringifyFunction} from '../../util/Function.js';
+import {ProtocolError, TimeoutError} from '../Errors.js';
+import {EventEmitter} from '../EventEmitter.js';
+import {PuppeteerLifeCycleEvent} from '../LifecycleWatcher.js';
+import {TimeoutSettings} from '../TimeoutSettings.js';
+import {EvaluateFunc, HandleFor} from '../types.js';
+import {
+  PuppeteerURL,
+  getSourcePuppeteerURLIfAvailable,
+  isString,
+  setPageContent,
+  waitWithTimeout,
+} from '../util.js';
+
+import {Connection} from './Connection.js';
+import {ElementHandle} from './ElementHandle.js';
+import {JSHandle} from './JSHandle.js';
+import {BidiSerializer} from './Serializer.js';
+import {createEvaluationError} from './utils.js';
+
+const SOURCE_URL_REGEX = /^[\040\t]*\/\/[@#] sourceURL=\s*(\S*?)\s*$/m;
+
+const getSourceUrlComment = (url: string) => {
+  return `//# sourceURL=${url}`;
+};
+
+/**
+ * @internal
+ */
+const lifeCycleToSubscribedEvent = new Map<PuppeteerLifeCycleEvent, string>([
+  ['load', 'browsingContext.load'],
+  ['domcontentloaded', 'browsingContext.domContentLoaded'],
+]);
+
+/**
+ * @internal
+ */
+const lifeCycleToReadinessState = new Map<
+  PuppeteerLifeCycleEvent,
+  Bidi.BrowsingContext.ReadinessState
+>([
+  ['load', 'complete'],
+  ['domcontentloaded', 'interactive'],
+]);
+
+/**
+ * @internal
+ */
+export class BrowsingContext extends EventEmitter {
+  connection: Connection;
+  #timeoutSettings: TimeoutSettings;
+  #id: string;
+  #url: string;
+
+  constructor(
+    connection: Connection,
+    timeoutSettings: TimeoutSettings,
+    info: Bidi.BrowsingContext.Info
+  ) {
+    super();
+    this.connection = connection;
+    this.#timeoutSettings = timeoutSettings;
+    this.#id = info.context;
+    this.#url = info.url;
+  }
+
+  get url(): string {
+    return this.#url;
+  }
+
+  get id(): string {
+    return this.#id;
+  }
+
+  async goto(
+    url: string,
+    options: {
+      referer?: string;
+      referrerPolicy?: string;
+      timeout?: number;
+      waitUntil?: PuppeteerLifeCycleEvent | PuppeteerLifeCycleEvent[];
+    } = {}
+  ): Promise<string | null> {
+    const {
+      waitUntil = 'load',
+      timeout = this.#timeoutSettings.navigationTimeout(),
+    } = options;
+
+    const readinessState = lifeCycleToReadinessState.get(
+      getWaitUntilSingle(waitUntil)
+    ) as Bidi.BrowsingContext.ReadinessState;
+
+    try {
+      const {result} = await waitWithTimeout(
+        this.connection.send('browsingContext.navigate', {
+          url: url,
+          context: this.#id,
+          wait: readinessState,
+        }),
+        'Navigation',
+        timeout
+      );
+      this.#url = result.url;
+
+      return result.navigation;
+    } catch (error) {
+      if (error instanceof ProtocolError) {
+        error.message += ` at ${url}`;
+      } else if (error instanceof TimeoutError) {
+        error.message = 'Navigation timeout of ' + timeout + ' ms exceeded';
+      }
+      throw error;
+    }
+  }
+
+  async evaluateHandle<
+    Params extends unknown[],
+    Func extends EvaluateFunc<Params> = EvaluateFunc<Params>
+  >(
+    pageFunction: Func | string,
+    ...args: Params
+  ): Promise<HandleFor<Awaited<ReturnType<Func>>>> {
+    return this.#evaluate(false, pageFunction, ...args);
+  }
+
+  async evaluate<
+    Params extends unknown[],
+    Func extends EvaluateFunc<Params> = EvaluateFunc<Params>
+  >(
+    pageFunction: Func | string,
+    ...args: Params
+  ): Promise<Awaited<ReturnType<Func>>> {
+    return this.#evaluate(true, pageFunction, ...args);
+  }
+
+  async #evaluate<
+    Params extends unknown[],
+    Func extends EvaluateFunc<Params> = EvaluateFunc<Params>
+  >(
+    returnByValue: true,
+    pageFunction: Func | string,
+    ...args: Params
+  ): Promise<Awaited<ReturnType<Func>>>;
+  async #evaluate<
+    Params extends unknown[],
+    Func extends EvaluateFunc<Params> = EvaluateFunc<Params>
+  >(
+    returnByValue: false,
+    pageFunction: Func | string,
+    ...args: Params
+  ): Promise<HandleFor<Awaited<ReturnType<Func>>>>;
+  async #evaluate<
+    Params extends unknown[],
+    Func extends EvaluateFunc<Params> = EvaluateFunc<Params>
+  >(
+    returnByValue: boolean,
+    pageFunction: Func | string,
+    ...args: Params
+  ): Promise<HandleFor<Awaited<ReturnType<Func>>> | Awaited<ReturnType<Func>>> {
+    const sourceUrlComment = getSourceUrlComment(
+      getSourcePuppeteerURLIfAvailable(pageFunction)?.toString() ??
+        PuppeteerURL.INTERNAL_URL
+    );
+
+    let responsePromise;
+    const resultOwnership = returnByValue ? 'none' : 'root';
+    if (isString(pageFunction)) {
+      const expression = SOURCE_URL_REGEX.test(pageFunction)
+        ? pageFunction
+        : `${pageFunction}\n${sourceUrlComment}\n`;
+
+      responsePromise = this.connection.send('script.evaluate', {
+        expression,
+        target: {context: this.#id},
+        resultOwnership,
+        awaitPromise: true,
+      });
+    } else {
+      let functionDeclaration = stringifyFunction(pageFunction);
+      functionDeclaration = SOURCE_URL_REGEX.test(functionDeclaration)
+        ? functionDeclaration
+        : `${functionDeclaration}\n${sourceUrlComment}\n`;
+      responsePromise = this.connection.send('script.callFunction', {
+        functionDeclaration,
+        arguments: await Promise.all(
+          args.map(arg => {
+            return BidiSerializer.serialize(arg, this);
+          })
+        ),
+        target: {context: this.#id},
+        resultOwnership,
+        awaitPromise: true,
+      });
+    }
+
+    const {result} = await responsePromise;
+
+    if ('type' in result && result.type === 'exception') {
+      throw createEvaluationError(result.exceptionDetails);
+    }
+
+    return returnByValue
+      ? BidiSerializer.deserialize(result.result)
+      : getBidiHandle(this, result.result);
+  }
+
+  async setContent(
+    html: string,
+    options: {
+      timeout?: number;
+      waitUntil?: PuppeteerLifeCycleEvent | PuppeteerLifeCycleEvent[];
+    }
+  ): Promise<void> {
+    const {
+      waitUntil = 'load',
+      timeout = this.#timeoutSettings.navigationTimeout(),
+    } = options;
+
+    const waitUntilCommand = lifeCycleToSubscribedEvent.get(
+      getWaitUntilSingle(waitUntil)
+    ) as string;
+
+    await Promise.all([
+      setPageContent(this, html),
+      waitWithTimeout(
+        new Promise<void>(resolve => {
+          this.once(waitUntilCommand, () => {
+            resolve();
+          });
+        }),
+        waitUntilCommand,
+        timeout
+      ),
+    ]);
+  }
+
+  async content(): Promise<string> {
+    return await this.evaluate(() => {
+      let retVal = '';
+      if (document.doctype) {
+        retVal = new XMLSerializer().serializeToString(document.doctype);
+      }
+      if (document.documentElement) {
+        retVal += document.documentElement.outerHTML;
+      }
+      return retVal;
+    });
+  }
+
+  async sendCDPCommand(
+    method: keyof ProtocolMapping.Commands,
+    params: object = {}
+  ): Promise<unknown> {
+    const session = await this.connection.send('cdp.getSession', {
+      context: this.#id,
+    });
+    // TODO: remove any once chromium-bidi types are updated.
+    const sessionId = (session.result as any).cdpSession;
+    return await this.connection.send('cdp.sendCommand', {
+      cdpMethod: method,
+      cdpParams: params,
+      cdpSession: sessionId,
+    });
+  }
+
+  dispose(): void {
+    this.removeAllListeners();
+    this.connection.unregisterBrowsingContexts(this.#id);
+  }
+}
+
+/**
+ * @internal
+ */
+export function getBidiHandle(
+  context: BrowsingContext,
+  result: Bidi.CommonDataTypes.RemoteValue
+): JSHandle | ElementHandle<Node> {
+  if (result.type === 'node' || result.type === 'window') {
+    return new ElementHandle(context, result);
+  }
+  return new JSHandle(context, result);
+}
+
+/**
+ * @internal
+ */
+export function getWaitUntilSingle(
+  event: PuppeteerLifeCycleEvent | PuppeteerLifeCycleEvent[]
+): Extract<PuppeteerLifeCycleEvent, 'load' | 'domcontentloaded'> {
+  if (Array.isArray(event) && event.length > 1) {
+    throw new Error('BiDi support only single `waitUntil` argument');
+  }
+  const waitUntilSingle = Array.isArray(event)
+    ? (event.find(lifecycle => {
+        return lifecycle === 'domcontentloaded' || lifecycle === 'load';
+      }) as PuppeteerLifeCycleEvent)
+    : event;
+
+  if (
+    waitUntilSingle === 'networkidle0' ||
+    waitUntilSingle === 'networkidle2'
+  ) {
+    throw new Error(`BiDi does not support 'waitUntil' ${waitUntilSingle}`);
+  }
+
+  assert(waitUntilSingle, `Invalid waitUntil option ${waitUntilSingle}`);
+
+  return waitUntilSingle;
+}

packages/puppeteer-core/src/common/bidi/Connection.ts

@@ -21,7 +21,7 @@ import {ConnectionTransport} from '../ConnectionTransport.js';
 import {debug} from '../Debug.js';
 import {EventEmitter} from '../EventEmitter.js';
 
-import {Context} from './Context.js';
+import {BrowsingContext} from './BrowsingContext.js';
 
 const debugProtocolSend = debug('puppeteer:webDriverBiDi:SEND ►');
 const debugProtocolReceive = debug('puppeteer:webDriverBiDi:RECV ◀');
@@ -103,7 +103,7 @@ export class Connection extends EventEmitter {
   #timeout? = 0;
   #closed = false;
   #callbacks = new CallbackRegistry();
-  #contexts: Map<string, Context> = new Map();
+  #browsingContexts: Map<string, BrowsingContext> = new Map();
 
   constructor(transport: ConnectionTransport, delay = 0, timeout?: number) {
     super();
@@ -119,10 +119,6 @@ export class Connection extends EventEmitter {
     return this.#closed;
   }
 
-  context(contextId: string): Context | null {
-    return this.#contexts.get(contextId) || null;
-  }
-
   send<T extends keyof Commands>(
     method: T,
     params: Commands[T]['params']
@@ -169,23 +165,23 @@ export class Connection extends EventEmitter {
   }
 
   #maybeEmitOnContext(event: Bidi.Message.EventMessage) {
-    let context: Context | undefined;
+    let context: BrowsingContext | undefined;
     // Context specific events
     if ('context' in event.params && event.params.context) {
-      context = this.#contexts.get(event.params.context);
+      context = this.#browsingContexts.get(event.params.context);
       // `log.entryAdded` specific context
     } else if ('source' in event.params && event.params.source.context) {
-      context = this.#contexts.get(event.params.source.context);
+      context = this.#browsingContexts.get(event.params.source.context);
     }
     context?.emit(event.method, event.params);
   }
 
-  registerContext(context: Context): void {
+  registerBrowsingContexts(context: BrowsingContext): void {
-    this.#contexts.set(context.id, context);
+    this.#browsingContexts.set(context.id, context);
   }
 
-  unregisterContext(context: Context): void {
+  unregisterBrowsingContexts(id: string): void {
-    this.#contexts.delete(context.id);
+    this.#browsingContexts.delete(id);
   }
 
   #onClose(): void {

packages/puppeteer-core/src/common/bidi/Context.ts

@@ -1,339 +0,0 @@
-/**
- * Copyright 2023 Google Inc. All rights reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import * as Bidi from 'chromium-bidi/lib/cjs/protocol/protocol.js';
-
-import {HTTPResponse} from '../../api/HTTPResponse.js';
-import {WaitForOptions} from '../../api/Page.js';
-import {assert} from '../../util/assert.js';
-import {stringifyFunction} from '../../util/Function.js';
-import {ProtocolMapping} from '../Connection.js';
-import {ProtocolError, TimeoutError} from '../Errors.js';
-import {EventEmitter} from '../EventEmitter.js';
-import {PuppeteerLifeCycleEvent} from '../LifecycleWatcher.js';
-import {EvaluateFunc, HandleFor} from '../types.js';
-import {
-  getSourcePuppeteerURLIfAvailable,
-  isString,
-  PuppeteerURL,
-  setPageContent,
-  waitWithTimeout,
-} from '../util.js';
-
-import {Connection} from './Connection.js';
-import {ElementHandle} from './ElementHandle.js';
-import {FrameManager} from './FrameManager.js';
-import {JSHandle} from './JSHandle.js';
-import {BidiSerializer} from './Serializer.js';
-import {createEvaluationError} from './utils.js';
-
-const SOURCE_URL_REGEX = /^[\040\t]*\/\/[@#] sourceURL=\s*(\S*?)\s*$/m;
-
-const getSourceUrlComment = (url: string) => {
-  return `//# sourceURL=${url}`;
-};
-
-/**
- * @internal
- */
-const lifeCycleToReadinessState = new Map<
-  PuppeteerLifeCycleEvent,
-  Bidi.BrowsingContext.ReadinessState
->([
-  ['load', 'complete'],
-  ['domcontentloaded', 'interactive'],
-]);
-
-/**
- * @internal
- */
-const lifeCycleToSubscribedEvent = new Map<PuppeteerLifeCycleEvent, string>([
-  ['load', 'browsingContext.load'],
-  ['domcontentloaded', 'browsingContext.domContentLoaded'],
-]);
-
-/**
- * @internal
- */
-export class Context extends EventEmitter {
-  #connection: Connection;
-  #url: string;
-  #id: string;
-  #parentId?: string | null;
-  _frameManager: FrameManager;
-
-  constructor(
-    connection: Connection,
-    frameManager: FrameManager,
-    result: Bidi.BrowsingContext.Info
-  ) {
-    super();
-    this.#connection = connection;
-    this._frameManager = frameManager;
-    this.#id = result.context;
-    this.#parentId = result.parent;
-    this.#url = result.url;
-
-    this.on(
-      'browsingContext.fragmentNavigated',
-      (info: Bidi.BrowsingContext.NavigationInfo) => {
-        this.#url = info.url;
-      }
-    );
-  }
-
-  get connection(): Connection {
-    return this.#connection;
-  }
-
-  get id(): string {
-    return this.#id;
-  }
-
-  get parentId(): string | undefined | null {
-    return this.#parentId;
-  }
-
-  async evaluateHandle<
-    Params extends unknown[],
-    Func extends EvaluateFunc<Params> = EvaluateFunc<Params>
-  >(
-    pageFunction: Func | string,
-    ...args: Params
-  ): Promise<HandleFor<Awaited<ReturnType<Func>>>> {
-    return this.#evaluate(false, pageFunction, ...args);
-  }
-
-  async evaluate<
-    Params extends unknown[],
-    Func extends EvaluateFunc<Params> = EvaluateFunc<Params>
-  >(
-    pageFunction: Func | string,
-    ...args: Params
-  ): Promise<Awaited<ReturnType<Func>>> {
-    return this.#evaluate(true, pageFunction, ...args);
-  }
-
-  async #evaluate<
-    Params extends unknown[],
-    Func extends EvaluateFunc<Params> = EvaluateFunc<Params>
-  >(
-    returnByValue: true,
-    pageFunction: Func | string,
-    ...args: Params
-  ): Promise<Awaited<ReturnType<Func>>>;
-  async #evaluate<
-    Params extends unknown[],
-    Func extends EvaluateFunc<Params> = EvaluateFunc<Params>
-  >(
-    returnByValue: false,
-    pageFunction: Func | string,
-    ...args: Params
-  ): Promise<HandleFor<Awaited<ReturnType<Func>>>>;
-  async #evaluate<
-    Params extends unknown[],
-    Func extends EvaluateFunc<Params> = EvaluateFunc<Params>
-  >(
-    returnByValue: boolean,
-    pageFunction: Func | string,
-    ...args: Params
-  ): Promise<HandleFor<Awaited<ReturnType<Func>>> | Awaited<ReturnType<Func>>> {
-    const sourceUrlComment = getSourceUrlComment(
-      getSourcePuppeteerURLIfAvailable(pageFunction)?.toString() ??
-        PuppeteerURL.INTERNAL_URL
-    );
-
-    let responsePromise;
-    const resultOwnership = returnByValue ? 'none' : 'root';
-    if (isString(pageFunction)) {
-      const expression = SOURCE_URL_REGEX.test(pageFunction)
-        ? pageFunction
-        : `${pageFunction}\n${sourceUrlComment}\n`;
-
-      responsePromise = this.#connection.send('script.evaluate', {
-        expression,
-        target: {context: this.#id},
-        resultOwnership,
-        awaitPromise: true,
-      });
-    } else {
-      let functionDeclaration = stringifyFunction(pageFunction);
-      functionDeclaration = SOURCE_URL_REGEX.test(functionDeclaration)
-        ? functionDeclaration
-        : `${functionDeclaration}\n${sourceUrlComment}\n`;
-      responsePromise = this.#connection.send('script.callFunction', {
-        functionDeclaration,
-        arguments: await Promise.all(
-          args.map(arg => {
-            return BidiSerializer.serialize(arg, this);
-          })
-        ),
-        target: {context: this.#id},
-        resultOwnership,
-        awaitPromise: true,
-      });
-    }
-
-    const {result} = await responsePromise;
-
-    if ('type' in result && result.type === 'exception') {
-      throw createEvaluationError(result.exceptionDetails);
-    }
-
-    return returnByValue
-      ? BidiSerializer.deserialize(result.result)
-      : getBidiHandle(this, result.result);
-  }
-
-  async goto(
-    url: string,
-    options: WaitForOptions & {
-      referer?: string | undefined;
-      referrerPolicy?: string | undefined;
-    } = {}
-  ): Promise<HTTPResponse | null> {
-    const {
-      waitUntil = 'load',
-      timeout = this._frameManager._timeoutSettings.navigationTimeout(),
-    } = options;
-
-    const readinessState = lifeCycleToReadinessState.get(
-      getWaitUntilSingle(waitUntil)
-    ) as Bidi.BrowsingContext.ReadinessState;
-
-    try {
-      const response = await waitWithTimeout(
-        this.connection.send('browsingContext.navigate', {
-          url: url,
-          context: this.id,
-          wait: readinessState,
-        }),
-        'Navigation',
-        timeout
-      );
-      this.#url = response.result.url;
-
-      return null;
-    } catch (error) {
-      if (error instanceof ProtocolError) {
-        error.message += ` at ${url}`;
-      } else if (error instanceof TimeoutError) {
-        error.message = 'Navigation timeout of ' + timeout + ' ms exceeded';
-      }
-      throw error;
-    }
-  }
-
-  url(): string {
-    return this.#url;
-  }
-
-  async setContent(
-    html: string,
-    options: WaitForOptions | undefined = {}
-  ): Promise<void> {
-    const {
-      waitUntil = 'load',
-      timeout = this._frameManager._timeoutSettings.navigationTimeout(),
-    } = options;
-
-    const waitUntilCommand = lifeCycleToSubscribedEvent.get(
-      getWaitUntilSingle(waitUntil)
-    ) as string;
-
-    await Promise.all([
-      setPageContent(this, html),
-      waitWithTimeout(
-        new Promise<void>(resolve => {
-          this.once(waitUntilCommand, () => {
-            resolve();
-          });
-        }),
-        waitUntilCommand,
-        timeout
-      ),
-    ]);
-  }
-
-  async content(): Promise<string> {
-    return await this.evaluate(() => {
-      let retVal = '';
-      if (document.doctype) {
-        retVal = new XMLSerializer().serializeToString(document.doctype);
-      }
-      if (document.documentElement) {
-        retVal += document.documentElement.outerHTML;
-      }
-      return retVal;
-    });
-  }
-
-  async sendCDPCommand(
-    method: keyof ProtocolMapping.Commands,
-    params: object = {}
-  ): Promise<unknown> {
-    const session = await this.#connection.send('cdp.getSession', {
-      context: this.id,
-    });
-    // TODO: remove any once chromium-bidi types are updated.
-    const sessionId = (session.result as any).cdpSession;
-    return await this.#connection.send('cdp.sendCommand', {
-      cdpMethod: method,
-      cdpParams: params,
-      cdpSession: sessionId,
-    });
-  }
-}
-
-/**
- * @internal
- */
-function getWaitUntilSingle(
-  event: PuppeteerLifeCycleEvent | PuppeteerLifeCycleEvent[]
-): Extract<PuppeteerLifeCycleEvent, 'load' | 'domcontentloaded'> {
-  if (Array.isArray(event) && event.length > 1) {
-    throw new Error('BiDi support only single `waitUntil` argument');
-  }
-  const waitUntilSingle = Array.isArray(event)
-    ? (event.find(lifecycle => {
-        return lifecycle === 'domcontentloaded' || lifecycle === 'load';
-      }) as PuppeteerLifeCycleEvent)
-    : event;
-
-  if (
-    waitUntilSingle === 'networkidle0' ||
-    waitUntilSingle === 'networkidle2'
-  ) {
-    throw new Error(`BiDi does not support 'waitUntil' ${waitUntilSingle}`);
-  }
-
-  assert(waitUntilSingle, `Invalid waitUntil option ${waitUntilSingle}`);
-
-  return waitUntilSingle;
-}
-
-/**
- * @internal
- */
-export function getBidiHandle(
-  context: Context,
-  result: Bidi.CommonDataTypes.RemoteValue
-): JSHandle | ElementHandle<Node> {
-  if (result.type === 'node' || result.type === 'window') {
-    return new ElementHandle(context, result);
-  }
-  return new JSHandle(context, result);
-}

packages/puppeteer-core/src/common/bidi/ElementHandle.ts

@@ -18,8 +18,7 @@ import * as Bidi from 'chromium-bidi/lib/cjs/protocol/protocol.js';
 
 import {ElementHandle as BaseElementHandle} from '../../api/ElementHandle.js';
 
-import {Connection} from './Connection.js';
+import {BrowsingContext} from './BrowsingContext.js';
-import {Context} from './Context.js';
 import {JSHandle} from './JSHandle.js';
 
 /**
@@ -30,18 +29,17 @@ export class ElementHandle<
 > extends BaseElementHandle<ElementType> {
   declare handle: JSHandle<ElementType>;
 
-  constructor(context: Context, remoteValue: Bidi.CommonDataTypes.RemoteValue) {
+  constructor(
+    context: BrowsingContext,
+    remoteValue: Bidi.CommonDataTypes.RemoteValue
+  ) {
     super(new JSHandle(context, remoteValue));
   }
 
-  context(): Context {
+  context(): BrowsingContext {
     return this.handle.context();
   }
 
-  get connection(): Connection {
-    return this.handle.connection;
-  }
-
   get isPrimitiveValue(): boolean {
     return this.handle.isPrimitiveValue;
   }

packages/puppeteer-core/src/common/bidi/Frame.ts

@@ -15,54 +15,48 @@
  */
 
 import {Frame as BaseFrame} from '../../api/Frame.js';
-import {HTTPResponse} from '../../api/HTTPResponse.js';
 import {PuppeteerLifeCycleEvent} from '../LifecycleWatcher.js';
 import {EvaluateFunc, HandleFor} from '../types.js';
 
-import {Context} from './Context.js';
+import {BrowsingContext} from './BrowsingContext.js';
-import {FrameManager} from './FrameManager.js';
+import {HTTPResponse} from './HTTPResponse.js';
 import {Page} from './Page.js';
 
 /**
+ * Puppeteer's Frame class could be viewed as a BiDi BrowsingContext implementation
  * @internal
  */
 export class Frame extends BaseFrame {
-  _frameManager: FrameManager;
+  #page: Page;
-  _context: Context;
+  #context: BrowsingContext;
+  override _id: string;
 
-  /**
+  constructor(page: Page, context: BrowsingContext, parentId?: string | null) {
-   * @internal
-   */
-  constructor(frameManager: FrameManager, context: Context) {
     super();
-    this._frameManager = frameManager;
+    this.#page = page;
-    this._context = context;
+    this.#context = context;
-    this._id = context.id;
+    this._id = this.#context.id;
-    this._parentId = context.parentId ?? undefined;
+    this._parentId = parentId ?? undefined;
   }
 
   override page(): Page {
-    return this._frameManager.page();
+    return this.#page;
   }
 
-  override async goto(
+  override name(): string {
-    url: string,
+    return this._name || '';
-    options?: {
-      referer?: string;
-      referrerPolicy?: string;
-      timeout?: number;
-      waitUntil?: PuppeteerLifeCycleEvent | PuppeteerLifeCycleEvent[];
-    }
-  ): Promise<HTTPResponse | null> {
-    return this._context.goto(url, options);
   }
 
-  override async waitForNavigation(options?: {
+  override url(): string {
-    timeout?: number;
+    return this.#context.url;
-    waitUntil?: PuppeteerLifeCycleEvent | PuppeteerLifeCycleEvent[];
+  }
-  }): Promise<HTTPResponse | null>;
+
-  override async waitForNavigation(): Promise<HTTPResponse | null> {
+  override parentFrame(): Frame | null {
-    throw new Error('Not implemented');
+    return this.#page.frame(this._parentId ?? '');
+  }
+
+  override childFrames(): Frame[] {
+    return this.#page.childFrames(this.#context.id);
   }
 
   override async evaluateHandle<
@@ -72,7 +66,7 @@ export class Frame extends BaseFrame {
     pageFunction: Func | string,
     ...args: Params
   ): Promise<HandleFor<Awaited<ReturnType<Func>>>> {
-    return this._context.evaluateHandle(pageFunction, ...args);
+    return this.#context.evaluateHandle(pageFunction, ...args);
   }
 
   override async evaluate<
@@ -82,44 +76,46 @@ export class Frame extends BaseFrame {
     pageFunction: Func | string,
     ...args: Params
   ): Promise<Awaited<ReturnType<Func>>> {
-    return this._context.evaluate(pageFunction, ...args);
+    return this.#context.evaluate(pageFunction, ...args);
   }
 
-  override async content(): Promise<string> {
+  override async goto(
-    return this._context.content();
+    url: string,
+    options?:
+      | {
+          referer?: string | undefined;
+          referrerPolicy?: string | undefined;
+          timeout?: number | undefined;
+          waitUntil?:
+            | PuppeteerLifeCycleEvent
+            | PuppeteerLifeCycleEvent[]
+            | undefined;
+        }
+      | undefined
+  ): Promise<HTTPResponse | null> {
+    const navigationId = await this.#context.goto(url, options);
+    return this.#page.getNavigationResponse(navigationId);
   }
 
-  override async setContent(
+  override setContent(
     html: string,
     options: {
       timeout?: number;
       waitUntil?: PuppeteerLifeCycleEvent | PuppeteerLifeCycleEvent[];
     }
   ): Promise<void> {
-    return this._context.setContent(html, options);
+    return this.#context.setContent(html, options);
   }
 
-  override name(): string {
+  override content(): Promise<string> {
-    return this._name || '';
+    return this.#context.content();
   }
 
-  override url(): string {
+  context(): BrowsingContext {
-    return this._context.url();
+    return this.#context;
   }
 
-  override parentFrame(): Frame | null {
+  dispose(): void {
-    return this._frameManager._frameTree.parentFrame(this._id) ?? null;
+    this.#context.dispose();
-  }
-
-  override childFrames(): Frame[] {
-    return this._frameManager._frameTree.childFrames(this._id);
-  }
-
-  override isDetached(): boolean {
-    throw new Error('Not implemented');
-  }
-
-  override async title(): Promise<string> {
-    throw new Error('Not implemented');
   }
 }

packages/puppeteer-core/src/common/bidi/FrameManager.ts

@@ -1,156 +0,0 @@
-/**
- * Copyright 2017 Google Inc. All rights reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import * as Bidi from 'chromium-bidi/lib/cjs/protocol/protocol.js';
-
-import {assert} from '../../util/assert.js';
-import {EventEmitter, Handler} from '../EventEmitter.js';
-import {FrameManagerEmittedEvents} from '../FrameManager.js';
-import {FrameTree} from '../FrameTree.js';
-import {TimeoutSettings} from '../TimeoutSettings.js';
-
-import {Connection} from './Connection.js';
-import {Context} from './Context.js';
-import {Frame} from './Frame.js';
-import {Page} from './Page.js';
-
-/**
- * A frame manager manages the frames for a given {@link Page | page}.
- *
- * @internal
- */
-export class FrameManager extends EventEmitter {
-  #page: Page;
-  #connection: Connection;
-  _contextId: string;
-  _frameTree = new FrameTree<Frame>();
-  _timeoutSettings: TimeoutSettings;
-
-  get client(): Connection {
-    return this.#connection;
-  }
-
-  // TODO: switch string to (typeof Browser.events)[number]
-  #subscribedEvents = new Map<string, Handler<any>>([
-    ['browsingContext.contextCreated', this.#onFrameAttached.bind(this)],
-    ['browsingContext.contextDestroyed', this.#onFrameDetached.bind(this)],
-    ['browsingContext.fragmentNavigated', this.#onFrameNavigated.bind(this)],
-  ]);
-
-  constructor(
-    connection: Connection,
-    page: Page,
-    info: Bidi.BrowsingContext.Info,
-    timeoutSettings: TimeoutSettings
-  ) {
-    super();
-    this.#connection = connection;
-    this.#page = page;
-    this._contextId = info.context;
-    this._timeoutSettings = timeoutSettings;
-    this.#handleFrameTree(info);
-    for (const [event, subscriber] of this.#subscribedEvents) {
-      this.#connection.on(event, subscriber);
-    }
-  }
-
-  page(): Page {
-    return this.#page;
-  }
-
-  mainFrame(): Frame {
-    const mainFrame = this._frameTree.getMainFrame();
-    assert(mainFrame, 'Requesting main frame too early!');
-    return mainFrame;
-  }
-
-  frames(): Frame[] {
-    return Array.from(this._frameTree.frames());
-  }
-
-  frame(frameId: string): Frame | null {
-    return this._frameTree.getById(frameId) || null;
-  }
-
-  #handleFrameTree(info: Bidi.BrowsingContext.Info): void {
-    if (info) {
-      this.#onFrameAttached(info);
-    }
-
-    if (!info.children) {
-      return;
-    }
-
-    for (const child of info.children) {
-      this.#handleFrameTree(child);
-    }
-  }
-
-  #onFrameAttached(info: Bidi.BrowsingContext.Info): void {
-    if (
-      !this.frame(info.context) &&
-      (this.frame(info.parent ?? '') || !this._frameTree.getMainFrame())
-    ) {
-      const context = new Context(this.#connection, this, info);
-      this.#connection.registerContext(context);
-
-      const frame = new Frame(this, context);
-      this._frameTree.addFrame(frame);
-      this.emit(FrameManagerEmittedEvents.FrameAttached, frame);
-    }
-  }
-
-  async #onFrameNavigated(
-    info: Bidi.BrowsingContext.NavigationInfo
-  ): Promise<void> {
-    const frameId = info.context;
-
-    let frame = this._frameTree.getById(frameId);
-    // Detach all child frames first.
-    if (frame) {
-      for (const child of frame.childFrames()) {
-        this.#removeFramesRecursively(child);
-      }
-    }
-
-    frame = await this._frameTree.waitForFrame(frameId);
-    // frame._navigated(info);
-    this.emit(FrameManagerEmittedEvents.FrameNavigated, frame);
-  }
-
-  #onFrameDetached(info: Bidi.BrowsingContext.Info): void {
-    const frame = this.frame(info.context);
-
-    if (frame) {
-      this.#removeFramesRecursively(frame);
-    }
-  }
-
-  #removeFramesRecursively(frame: Frame): void {
-    for (const child of frame.childFrames()) {
-      this.#removeFramesRecursively(child);
-    }
-    this.#connection.unregisterContext(frame._context);
-    this._frameTree.removeFrame(frame);
-    this.emit(FrameManagerEmittedEvents.FrameDetached, frame);
-  }
-
-  dispose(): void {
-    this.removeAllListeners();
-    for (const [event, subscriber] of this.#subscribedEvents) {
-      this.#connection.off(event, subscriber);
-    }
-  }
-}

packages/puppeteer-core/src/common/bidi/HTTPRequest.ts

@@ -0,0 +1,116 @@
+/**
+ * Copyright 2020 Google Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import * as Bidi from 'chromium-bidi/lib/cjs/protocol/protocol.js';
+
+import {Frame} from '../../api/Frame.js';
+import {
+  HTTPRequest as BaseHTTPRequest,
+  ResourceType,
+} from '../../api/HTTPRequest.js';
+
+import {HTTPResponse} from './HTTPResponse.js';
+
+/**
+ * @internal
+ */
+export class HTTPRequest extends BaseHTTPRequest {
+  override _response: HTTPResponse | null = null;
+  override _redirectChain: HTTPRequest[];
+  _navigationId: string | null;
+
+  #url: string;
+  #resourceType: ResourceType;
+
+  #method: string;
+  #postData?: string;
+  #headers: Record<string, string> = {};
+  #initiator: Bidi.Network.Initiator;
+  #frame: Frame | null;
+
+  constructor(
+    event: Bidi.Network.BeforeRequestSentParams,
+    frame: Frame | null,
+    redirectChain: HTTPRequest[]
+  ) {
+    super();
+
+    this.#url = event.request.url;
+    this.#resourceType = event.initiator.type.toLowerCase() as ResourceType;
+    this.#method = event.request.method;
+    this.#postData = undefined;
+    this.#initiator = event.initiator;
+    this.#frame = frame;
+
+    this._requestId = event.request.request;
+    this._redirectChain = redirectChain ?? [];
+    this._navigationId = event.navigation;
+
+    for (const {name, value} of event.request.headers) {
+      // TODO: How to handle Binary Headers
+      // https://w3c.github.io/webdriver-bidi/#type-network-Header
+      if (value) {
+        this.#headers[name.toLowerCase()] = value;
+      }
+    }
+  }
+
+  override url(): string {
+    return this.#url;
+  }
+
+  override resourceType(): ResourceType {
+    return this.#resourceType;
+  }
+
+  override method(): string {
+    return this.#method;
+  }
+
+  override postData(): string | undefined {
+    return this.#postData;
+  }
+
+  override headers(): Record<string, string> {
+    return this.#headers;
+  }
+
+  override response(): HTTPResponse | null {
+    return this._response;
+  }
+
+  override isNavigationRequest(): boolean {
+    return Boolean(this._navigationId);
+  }
+
+  override initiator(): Bidi.Network.Initiator {
+    return this.#initiator;
+  }
+
+  override redirectChain(): HTTPRequest[] {
+    return this._redirectChain.slice();
+  }
+
+  override enqueueInterceptAction(
+    pendingHandler: () => void | PromiseLike<unknown>
+  ): void {
+    // Execute the handler when interception is not supported
+    void pendingHandler();
+  }
+
+  override frame(): Frame | null {
+    return this.#frame;
+  }
+}

packages/puppeteer-core/src/common/bidi/HTTPResponse.ts

@@ -0,0 +1,95 @@
+/**
+ * Copyright 2020 Google Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import * as Bidi from 'chromium-bidi/lib/cjs/protocol/protocol.js';
+import Protocol from 'devtools-protocol';
+
+import {
+  HTTPResponse as BaseHTTPResponse,
+  RemoteAddress,
+} from '../../api/HTTPResponse.js';
+
+import {HTTPRequest} from './HTTPRequest.js';
+
+/**
+ * @internal
+ */
+export class HTTPResponse extends BaseHTTPResponse {
+  #request: HTTPRequest;
+  #remoteAddress: RemoteAddress;
+  #status: number;
+  #statusText: string;
+  #url: string;
+  #fromCache: boolean;
+  #headers: Record<string, string> = {};
+  #timings: Record<string, string> | null;
+
+  constructor(
+    request: HTTPRequest,
+    responseEvent: Bidi.Network.ResponseCompletedParams
+  ) {
+    super();
+    const {response} = responseEvent;
+    this.#request = request;
+
+    this.#remoteAddress = {
+      ip: '',
+      port: -1,
+    };
+
+    this.#url = response.url;
+    this.#fromCache = response.fromCache;
+    this.#status = response.status;
+    this.#statusText = response.statusText;
+    // TODO: update once BiDi has types
+    this.#timings = (response as any).timings ?? null;
+
+    for (const header of response.headers) {
+      this.#headers[header.name] = header.value ?? '';
+    }
+  }
+
+  override remoteAddress(): RemoteAddress {
+    return this.#remoteAddress;
+  }
+
+  override url(): string {
+    return this.#url;
+  }
+
+  override status(): number {
+    return this.#status;
+  }
+
+  override statusText(): string {
+    return this.#statusText;
+  }
+
+  override headers(): Record<string, string> {
+    return this.#headers;
+  }
+
+  override request(): HTTPRequest {
+    return this.#request;
+  }
+
+  override fromCache(): boolean {
+    return this.#fromCache;
+  }
+
+  override timing(): Protocol.Network.ResourceTiming | null {
+    return this.#timings as any;
+  }
+}

packages/puppeteer-core/src/common/bidi/JSHandle.ts

@@ -21,8 +21,7 @@ import {JSHandle as BaseJSHandle} from '../../api/JSHandle.js';
 import {EvaluateFuncWith, HandleFor, HandleOr} from '../../common/types.js';
 import {withSourcePuppeteerURLIfNone} from '../util.js';
 
-import {Connection} from './Connection.js';
+import {BrowsingContext} from './BrowsingContext.js';
-import {Context} from './Context.js';
 import {BidiSerializer} from './Serializer.js';
 import {releaseReference} from './utils.js';
 
@@ -31,20 +30,19 @@ export class JSHandle<T = unknown> extends BaseJSHandle<T> {
   #context;
   #remoteValue;
 
-  constructor(context: Context, remoteValue: Bidi.CommonDataTypes.RemoteValue) {
+  constructor(
+    context: BrowsingContext,
+    remoteValue: Bidi.CommonDataTypes.RemoteValue
+  ) {
     super();
     this.#context = context;
     this.#remoteValue = remoteValue;
   }
 
-  context(): Context {
+  context(): BrowsingContext {
     return this.#context;
   }
 
-  get connection(): Connection {
-    return this.#context.connection;
-  }
-
   override get disposed(): boolean {
     return this.#disposed;
   }
@@ -74,7 +72,7 @@ export class JSHandle<T = unknown> extends BaseJSHandle<T> {
       this.evaluateHandle.name,
       pageFunction
     );
-    return await this.context().evaluateHandle(pageFunction, this, ...args);
+    return this.context().evaluateHandle(pageFunction, this, ...args);
   }
 
   override async getProperty<K extends keyof T>(

packages/puppeteer-core/src/common/bidi/NetworkManager.ts

@@ -0,0 +1,113 @@
+/**
+ * Copyright 2023 Google Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as Bidi from 'chromium-bidi/lib/cjs/protocol/protocol.js';
+
+import {EventEmitter, Handler} from '../EventEmitter.js';
+import {NetworkManagerEmittedEvents} from '../NetworkManager.js';
+
+import {Connection} from './Connection.js';
+import {HTTPRequest} from './HTTPRequest.js';
+import {HTTPResponse} from './HTTPResponse.js';
+import {Page} from './Page.js';
+
+/**
+ * @internal
+ */
+export class NetworkManager extends EventEmitter {
+  #connection: Connection;
+  #page: Page;
+  #subscribedEvents = new Map<string, Handler<any>>([
+    ['network.beforeRequestSent', this.#onBeforeRequestSent.bind(this)],
+    ['network.responseStarted', this.#onResponseStarted.bind(this)],
+    ['network.responseCompleted', this.#onResponseCompleted.bind(this)],
+    ['network.fetchError', this.#onFetchError.bind(this)],
+  ]) as Map<Bidi.Message.EventNames, Handler>;
+
+  #requestMap = new Map<string, HTTPRequest>();
+  #navigationMap = new Map<string, HTTPResponse>();
+
+  constructor(connection: Connection, page: Page) {
+    super();
+    this.#connection = connection;
+    this.#page = page;
+
+    // TODO: Subscribe to the Frame indivutally
+    for (const [event, subscriber] of this.#subscribedEvents) {
+      this.#connection.on(event, subscriber);
+    }
+  }
+
+  #onBeforeRequestSent(event: Bidi.Network.BeforeRequestSentParams): void {
+    const frame = this.#page.frame(event.context ?? '');
+    if (!frame) {
+      return;
+    }
+    const request = this.#requestMap.get(event.request.request);
+
+    let upsertRequest: HTTPRequest;
+    if (request) {
+      const requestChain = request._redirectChain;
+
+      upsertRequest = new HTTPRequest(event, frame, requestChain);
+    } else {
+      upsertRequest = new HTTPRequest(event, frame, []);
+    }
+    this.#requestMap.set(event.request.request, upsertRequest);
+    this.emit(NetworkManagerEmittedEvents.Request, upsertRequest);
+  }
+
+  #onResponseStarted(_event: any) {}
+
+  #onResponseCompleted(event: Bidi.Network.ResponseCompletedParams): void {
+    const request = this.#requestMap.get(event.request.request);
+    if (request) {
+      const response = new HTTPResponse(request, event);
+      request._response = response;
+      if (event.navigation) {
+        this.#navigationMap.set(event.navigation, response);
+      }
+      if (response.fromCache()) {
+        this.emit(NetworkManagerEmittedEvents.RequestServedFromCache, request);
+      }
+      this.emit(NetworkManagerEmittedEvents.Response, response);
+      this.emit(NetworkManagerEmittedEvents.RequestFinished, request);
+    }
+  }
+
+  #onFetchError(event: any) {
+    const request = this.#requestMap.get(event.request.request);
+    if (!request) {
+      return;
+    }
+    request._failureText = event.errorText;
+    this.emit(NetworkManagerEmittedEvents.RequestFailed, request);
+  }
+
+  getNavigationResponse(navigationId: string | null): HTTPResponse | null {
+    return this.#navigationMap.get(navigationId ?? '') ?? null;
+  }
+
+  dispose(): void {
+    this.removeAllListeners();
+    this.#requestMap.clear();
+    this.#navigationMap.clear();
+
+    for (const [event, subscriber] of this.#subscribedEvents) {
+      this.#connection.off(event, subscriber);
+    }
+  }
+}

packages/puppeteer-core/src/common/bidi/Page.ts

@@ -18,17 +18,20 @@ import type {Readable} from 'stream';
 
 import * as Bidi from 'chromium-bidi/lib/cjs/protocol/protocol.js';
 
-import {HTTPResponse} from '../../api/HTTPResponse.js';
 import {
   Page as PageBase,
   PageEmittedEvents,
   ScreenshotOptions,
   WaitForOptions,
 } from '../../api/Page.js';
+import {assert} from '../../util/assert.js';
 import {isErrorLike} from '../../util/ErrorLike.js';
+import {isTargetClosedError} from '../Connection.js';
 import {ConsoleMessage, ConsoleMessageLocation} from '../ConsoleMessage.js';
-import {EventType, Handler} from '../EventEmitter.js';
+import {Handler} from '../EventEmitter.js';
 import {FrameManagerEmittedEvents} from '../FrameManager.js';
+import {FrameTree} from '../FrameTree.js';
+import {NetworkManagerEmittedEvents} from '../NetworkManager.js';
 import {PDFOptions} from '../PDFOptions.js';
 import {Viewport} from '../PuppeteerViewport.js';
 import {TimeoutSettings} from '../TimeoutSettings.js';
@@ -39,43 +42,70 @@ import {
   withSourcePuppeteerURLIfNone,
 } from '../util.js';
 
+import {BrowsingContext, getBidiHandle} from './BrowsingContext.js';
 import {Connection} from './Connection.js';
-import {Context, getBidiHandle} from './Context.js';
 import {Frame} from './Frame.js';
-import {FrameManager} from './FrameManager.js';
+import {HTTPResponse} from './HTTPResponse.js';
+import {NetworkManager} from './NetworkManager.js';
 import {BidiSerializer} from './Serializer.js';
 
 /**
  * @internal
  */
 export class Page extends PageBase {
-  _timeoutSettings = new TimeoutSettings();
+  #timeoutSettings = new TimeoutSettings();
   #connection: Connection;
-  #frameManager: FrameManager;
+  #frameTree = new FrameTree<Frame>();
+  #networkManager: NetworkManager;
   #viewport: Viewport | null = null;
   #closed = false;
   #subscribedEvents = new Map<string, Handler<any>>([
     ['log.entryAdded', this.#onLogEntryAdded.bind(this)],
-    ['browsingContext.load', this.#onLoad.bind(this)],
+    [
-    ['browsingContext.domContentLoaded', this.#onDOMLoad.bind(this)],
+      'browsingContext.load',
+      () => {
+        return this.emit(PageEmittedEvents.Load);
+      },
+    ],
+    [
+      'browsingContext.domContentLoaded',
+      () => {
+        return this.emit(PageEmittedEvents.DOMContentLoaded);
+      },
+    ],
+    ['browsingContext.contextCreated', this.#onFrameAttached.bind(this)],
+    ['browsingContext.contextDestroyed', this.#onFrameDetached.bind(this)],
+    ['browsingContext.fragmentNavigated', this.#onFrameNavigated.bind(this)],
   ]) as Map<Bidi.Session.SubscriptionRequestEvent, Handler>;
-  #frameManagerEvents = new Map<EventType, Handler<any>>([
+  #networkManagerEvents = new Map<symbol, Handler<any>>([
     [
-      FrameManagerEmittedEvents.FrameAttached,
+      NetworkManagerEmittedEvents.Request,
-      frame => {
+      event => {
-        return this.emit(PageEmittedEvents.FrameAttached, frame);
+        return this.emit(PageEmittedEvents.Request, event);
       },
     ],
     [
-      FrameManagerEmittedEvents.FrameDetached,
+      NetworkManagerEmittedEvents.RequestServedFromCache,
-      frame => {
+      event => {
-        return this.emit(PageEmittedEvents.FrameDetached, frame);
+        return this.emit(PageEmittedEvents.RequestServedFromCache, event);
       },
     ],
     [
-      FrameManagerEmittedEvents.FrameNavigated,
+      NetworkManagerEmittedEvents.RequestFailed,
-      frame => {
+      event => {
-        return this.emit(PageEmittedEvents.FrameNavigated, frame);
+        return this.emit(PageEmittedEvents.RequestFailed, event);
+      },
+    ],
+    [
+      NetworkManagerEmittedEvents.RequestFinished,
+      event => {
+        return this.emit(PageEmittedEvents.RequestFinished, event);
+      },
+    ],
+    [
+      NetworkManagerEmittedEvents.Response,
+      event => {
+        return this.emit(PageEmittedEvents.Response, event);
       },
     ],
   ]);
@@ -84,15 +114,12 @@ export class Page extends PageBase {
     super();
     this.#connection = connection;
 
-    this.#frameManager = new FrameManager(
+    this.#networkManager = new NetworkManager(connection, this);
-      this.#connection,
-      this,
-      info,
-      this._timeoutSettings
-    );
 
-    for (const [event, subscriber] of this.#frameManagerEvents) {
+    this.#handleFrameTree(info);
-      this.#frameManager.on(event, subscriber);
+
+    for (const [event, subscriber] of this.#networkManagerEvents) {
+      this.#networkManager.on(event, subscriber);
     }
   }
 
@@ -103,16 +130,17 @@ export class Page extends PageBase {
     const page = new Page(connection, info);
 
     for (const [event, subscriber] of page.#subscribedEvents) {
-      page.context().on(event, subscriber);
+      connection.on(event, subscriber);
     }
 
     await page.#connection
       .send('session.subscribe', {
         events: [...page.#subscribedEvents.keys()],
+        // TODO: We should subscribe globally
         contexts: [info.context],
       })
       .catch(error => {
-        if (isErrorLike(error) && !error.message.includes('Target closed')) {
+        if (isErrorLike(error) && isTargetClosedError(error)) {
           throw error;
         }
       });
@@ -121,21 +149,93 @@ export class Page extends PageBase {
   }
 
   override mainFrame(): Frame {
-    return this.#frameManager.mainFrame();
+    const mainFrame = this.#frameTree.getMainFrame();
+    assert(mainFrame, 'Requesting main frame too early!');
+    return mainFrame;
   }
 
   override frames(): Frame[] {
-    return this.#frameManager.frames();
+    return Array.from(this.#frameTree.frames());
   }
 
-  context(): Context {
+  frame(frameId: string): Frame | null {
-    return this.#frameManager.mainFrame()._context;
+    return this.#frameTree.getById(frameId) || null;
+  }
+
+  childFrames(frameId: string): Frame[] {
+    return this.#frameTree.childFrames(frameId);
+  }
+
+  #handleFrameTree(info: Bidi.BrowsingContext.Info): void {
+    if (info) {
+      this.#onFrameAttached(info);
+    }
+
+    if (!info.children) {
+      return;
+    }
+
+    for (const child of info.children) {
+      this.#handleFrameTree(child);
+    }
+  }
+
+  #onFrameAttached(info: Bidi.BrowsingContext.Info): void {
+    if (
+      !this.frame(info.context) &&
+      (this.frame(info.parent ?? '') || !this.#frameTree.getMainFrame())
+    ) {
+      const context = new BrowsingContext(
+        this.#connection,
+        this.#timeoutSettings,
+        info
+      );
+      this.#connection.registerBrowsingContexts(context);
+      const frame = new Frame(this, context, info.parent);
+
+      this.#frameTree.addFrame(frame);
+      this.emit(FrameManagerEmittedEvents.FrameAttached, frame);
+    }
+  }
+
+  async #onFrameNavigated(
+    info: Bidi.BrowsingContext.NavigationInfo
+  ): Promise<void> {
+    const frameId = info.context;
+
+    let frame = this.frame(frameId);
+    // Detach all child frames first.
+    if (frame) {
+      for (const child of frame.childFrames()) {
+        this.#removeFramesRecursively(child);
+      }
+    }
+
+    frame = await this.#frameTree.waitForFrame(frameId);
+    this.emit(FrameManagerEmittedEvents.FrameNavigated, frame);
+  }
+
+  #onFrameDetached(info: Bidi.BrowsingContext.Info): void {
+    const frame = this.frame(info.context);
+
+    if (frame) {
+      this.#removeFramesRecursively(frame);
+    }
+  }
+
+  #removeFramesRecursively(frame: Frame): void {
+    for (const child of frame.childFrames()) {
+      this.#removeFramesRecursively(child);
+    }
+    frame.dispose();
+    this.#frameTree.removeFrame(frame);
+    this.emit(FrameManagerEmittedEvents.FrameDetached, frame);
   }
 
   #onLogEntryAdded(event: Bidi.Log.LogEntry): void {
     if (isConsoleLogEntry(event)) {
       const args = event.args.map(arg => {
-        return getBidiHandle(this.context(), arg);
+        return getBidiHandle(this.mainFrame().context(), arg);
       });
 
       const text = args
@@ -183,12 +283,8 @@ export class Page extends PageBase {
     }
   }
 
-  #onLoad(_event: Bidi.BrowsingContext.NavigationInfo): void {
+  getNavigationResponse(id: string | null): HTTPResponse | null {
-    this.emit(PageEmittedEvents.Load);
+    return this.#networkManager.getNavigationResponse(id);
-  }
-
-  #onDOMLoad(_event: Bidi.BrowsingContext.NavigationInfo): void {
-    this.emit(PageEmittedEvents.DOMContentLoaded);
   }
 
   override async close(): Promise<void> {
@@ -197,16 +293,13 @@ export class Page extends PageBase {
     }
     this.#closed = true;
     this.removeAllListeners();
-    this.#frameManager.dispose();
+    this.#networkManager.dispose();
-
-    for (const [event, subscriber] of this.#subscribedEvents) {
-      this.context().off(event, subscriber);
-    }
 
     await this.#connection
       .send('session.unsubscribe', {
         events: [...this.#subscribedEvents.keys()],
-        contexts: [this.context().id],
+        // TODO: Remove this once we subscrite gloablly
+        contexts: [this.mainFrame()._id],
       })
       .catch(() => {
         // Suppress the error as we remove the context
@@ -214,7 +307,7 @@ export class Page extends PageBase {
       });
 
     await this.#connection.send('browsingContext.close', {
-      context: this.context().id,
+      context: this.mainFrame()._id,
     });
   }
 
@@ -261,11 +354,15 @@ export class Page extends PageBase {
   }
 
   override setDefaultNavigationTimeout(timeout: number): void {
-    this._timeoutSettings.setDefaultNavigationTimeout(timeout);
+    this.#timeoutSettings.setDefaultNavigationTimeout(timeout);
   }
 
   override setDefaultTimeout(timeout: number): void {
-    this._timeoutSettings.setDefaultTimeout(timeout);
+    this.#timeoutSettings.setDefaultTimeout(timeout);
+  }
+
+  override getDefaultTimeout(): number {
+    return this.#timeoutSettings.timeout();
   }
 
   override async setContent(
@@ -276,16 +373,7 @@ export class Page extends PageBase {
   }
 
   override async content(): Promise<string> {
-    return await this.evaluate(() => {
+    return this.mainFrame().content();
-      let retVal = '';
-      if (document.doctype) {
-        retVal = new XMLSerializer().serializeToString(document.doctype);
-      }
-      if (document.documentElement) {
-        retVal += document.documentElement.outerHTML;
-      }
-      return retVal;
-    });
   }
 
   override async setViewport(viewport: Viewport): Promise<void> {
@@ -296,13 +384,15 @@ export class Page extends PageBase {
     const deviceScaleFactor = 1;
     const screenOrientation = {angle: 0, type: 'portraitPrimary'};
 
-    await this.context().sendCDPCommand('Emulation.setDeviceMetricsOverride', {
+    await this.mainFrame()
-      mobile,
+      .context()
-      width,
+      .sendCDPCommand('Emulation.setDeviceMetricsOverride', {
-      height,
+        mobile,
-      deviceScaleFactor,
+        width,
-      screenOrientation,
+        height,
-    });
+        deviceScaleFactor,
+        screenOrientation,
+      });
 
     this.#viewport = viewport;
   }
@@ -326,7 +416,7 @@ export class Page extends PageBase {
     } = this._getPDFOptions(options, 'cm');
     const {result} = await waitWithTimeout(
       this.#connection.send('browsingContext.print', {
-        context: this.context().id,
+        context: this.mainFrame()._id,
         background,
         margin,
         orientation: landscape ? 'landscape' : 'portrait',
@@ -383,7 +473,7 @@ export class Page extends PageBase {
     const {result} = await this.#connection.send(
       'browsingContext.captureScreenshot',
       {
-        context: this.context().id,
+        context: this.mainFrame()._id,
       }
     );
 

packages/puppeteer-core/src/common/bidi/Serializer.ts

@@ -18,7 +18,7 @@ import * as Bidi from 'chromium-bidi/lib/cjs/protocol/protocol.js';
 
 import {debugError, isDate, isPlainObject, isRegExp} from '../util.js';
 
-import {Context} from './Context.js';
+import {BrowsingContext} from './BrowsingContext.js';
 import {ElementHandle} from './ElementHandle.js';
 import {JSHandle} from './JSHandle.js';
 
@@ -143,7 +143,7 @@ export class BidiSerializer {
 
   static serialize(
     arg: unknown,
-    context: Context
+    context: BrowsingContext
   ): Bidi.CommonDataTypes.LocalValue | Bidi.CommonDataTypes.RemoteValue {
     // TODO: See use case of LazyArgs
     const objectHandle =

packages/puppeteer-core/src/common/bidi/utils.ts

@@ -19,7 +19,7 @@ import * as Bidi from 'chromium-bidi/lib/cjs/protocol/protocol.js';
 import {debug} from '../Debug.js';
 import {PuppeteerURL} from '../util.js';
 
-import {Context} from './Context.js';
+import {BrowsingContext} from './BrowsingContext.js';
 import {BidiSerializer} from './Serializer.js';
 
 /**
@@ -30,7 +30,7 @@ export const debugError = debug('puppeteer:error');
  * @internal
  */
 export async function releaseReference(
-  client: Context,
+  client: BrowsingContext,
   remoteReference: Bidi.CommonDataTypes.RemoteReference
 ): Promise<void> {
   if (!remoteReference.handle) {

test/TestExpectations.json

@@ -59,6 +59,48 @@
     "parameters": ["webDriverBiDi"],
     "expectations": ["SKIP"]
   },
+  {
+    "testIdPattern": "[network.spec] network *",
+    "platforms": ["darwin", "linux", "win32"],
+    "parameters": ["webDriverBiDi"],
+    "expectations": ["PASS"]
+  },
+  {
+    "testIdPattern": "[network.spec] network Network Events *",
+    "platforms": ["darwin", "linux", "win32"],
+    "parameters": ["webDriverBiDi"],
+    "expectations": ["FAIL"]
+  },
+  {
+    "testIdPattern": "[network.spec] network Page.authenticate *",
+    "platforms": ["darwin", "linux", "win32"],
+    "parameters": ["webDriverBiDi"],
+    "expectations": ["FAIL"]
+  },
+  {
+    "testIdPattern": "[network.spec] network Page.setExtraHTTPHeaders *",
+    "platforms": ["darwin", "linux", "win32"],
+    "parameters": ["webDriverBiDi"],
+    "expectations": ["FAIL"]
+  },
+  {
+    "testIdPattern": "[network.spec] network Response.buffer *",
+    "platforms": ["darwin", "linux", "win32"],
+    "parameters": ["webDriverBiDi"],
+    "expectations": ["FAIL", "TIMEOUT"]
+  },
+  {
+    "testIdPattern": "[network.spec] network Response.json *",
+    "platforms": ["darwin", "linux", "win32"],
+    "parameters": ["webDriverBiDi"],
+    "expectations": ["FAIL"]
+  },
+  {
+    "testIdPattern": "[network.spec] network Response.text *",
+    "platforms": ["darwin", "linux", "win32"],
+    "parameters": ["webDriverBiDi"],
+    "expectations": ["FAIL"]
+  },
   {
     "testIdPattern": "[NetworkManager.spec] NetworkManager *",
     "platforms": ["darwin", "linux", "win32"],
@@ -344,19 +386,7 @@
   {
     "testIdPattern": "[navigation.spec] navigation Page.goto should fail when navigating to bad SSL",
     "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["webDriverBiDi"],
+    "parameters": ["firefox"],
-    "expectations": ["FAIL"]
-  },
-  {
-    "testIdPattern": "[navigation.spec] navigation Page.goto should navigate to dataURL and fire dataURL requests",
-    "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["webDriverBiDi"],
-    "expectations": ["FAIL"]
-  },
-  {
-    "testIdPattern": "[navigation.spec] navigation Page.goto should navigate to empty page with domcontentloaded",
-    "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["webDriverBiDi"],
     "expectations": ["FAIL"]
   },
   {
@@ -371,36 +401,6 @@
     "parameters": ["webDriverBiDi"],
     "expectations": ["FAIL"]
   },
-  {
-    "testIdPattern": "[navigation.spec] navigation Page.goto should navigate to URL with hash and fire requests without hash",
-    "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["webDriverBiDi"],
-    "expectations": ["FAIL", "TIMEOUT"]
-  },
-  {
-    "testIdPattern": "[navigation.spec] navigation Page.goto should not throw an error for a 404 response with an empty body",
-    "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["webDriverBiDi"],
-    "expectations": ["FAIL"]
-  },
-  {
-    "testIdPattern": "[navigation.spec] navigation Page.goto should not throw an error for a 500 response with an empty body",
-    "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["webDriverBiDi"],
-    "expectations": ["FAIL"]
-  },
-  {
-    "testIdPattern": "[navigation.spec] navigation Page.goto should return last response in redirect chain",
-    "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["webDriverBiDi"],
-    "expectations": ["FAIL", "TIMEOUT"]
-  },
-  {
-    "testIdPattern": "[navigation.spec] navigation Page.goto should return response when page changes its URL after load",
-    "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["webDriverBiDi"],
-    "expectations": ["FAIL"]
-  },
   {
     "testIdPattern": "[navigation.spec] navigation Page.goto should send referer",
     "platforms": ["darwin", "linux", "win32"],
@@ -413,42 +413,114 @@
     "parameters": ["webDriverBiDi"],
     "expectations": ["FAIL"]
   },
-  {
-    "testIdPattern": "[navigation.spec] navigation Page.goto should work when navigating to 404",
-    "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["webDriverBiDi"],
-    "expectations": ["FAIL"]
-  },
-  {
-    "testIdPattern": "[navigation.spec] navigation Page.goto should work when navigating to data url",
-    "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["webDriverBiDi"],
-    "expectations": ["FAIL"]
-  },
-  {
-    "testIdPattern": "[navigation.spec] navigation Page.goto should work when navigating to valid url",
-    "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["webDriverBiDi"],
-    "expectations": ["FAIL", "TIMEOUT"]
-  },
-  {
-    "testIdPattern": "[navigation.spec] navigation Page.goto should work when page calls history API in beforeunload",
-    "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["webDriverBiDi"],
-    "expectations": ["FAIL"]
-  },
-  {
-    "testIdPattern": "[navigation.spec] navigation Page.goto should work with self requesting page",
-    "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["webDriverBiDi"],
-    "expectations": ["FAIL"]
-  },
   {
     "testIdPattern": "[navigation.spec] navigation Page.reload should work",
     "platforms": ["darwin", "linux", "win32"],
     "parameters": ["webDriverBiDi"],
     "expectations": ["FAIL", "TIMEOUT"]
   },
+  {
+    "testIdPattern": "[network.spec] network Network Events Page.Events.RequestFinished",
+    "platforms": ["darwin", "linux", "win32"],
+    "parameters": ["webDriverBiDi"],
+    "expectations": ["PASS"]
+  },
+  {
+    "testIdPattern": "[network.spec] network Network Events should fire events in proper order",
+    "platforms": ["darwin", "linux", "win32"],
+    "parameters": ["webDriverBiDi"],
+    "expectations": ["PASS"]
+  },
+  {
+    "testIdPattern": "[network.spec] network Page.Events.Request should fire for iframes",
+    "platforms": ["darwin", "linux", "win32"],
+    "parameters": ["webDriverBiDi"],
+    "expectations": ["FAIL"]
+  },
+  {
+    "testIdPattern": "[network.spec] network raw network headers Cross-origin set-cookie",
+    "platforms": ["darwin", "linux", "win32"],
+    "parameters": ["webDriverBiDi"],
+    "expectations": ["FAIL"]
+  },
+  {
+    "testIdPattern": "[network.spec] network raw network headers Same-origin set-cookie navigation",
+    "platforms": ["darwin", "linux", "win32"],
+    "parameters": ["webDriverBiDi"],
+    "expectations": ["FAIL"]
+  },
+  {
+    "testIdPattern": "[network.spec] network raw network headers Same-origin set-cookie subresource",
+    "platforms": ["darwin", "linux", "win32"],
+    "parameters": ["webDriverBiDi"],
+    "expectations": ["FAIL"]
+  },
+  {
+    "testIdPattern": "[network.spec] network Request.frame should work for subframe navigation request",
+    "platforms": ["darwin", "linux", "win32"],
+    "parameters": ["webDriverBiDi"],
+    "expectations": ["FAIL"]
+  },
+  {
+    "testIdPattern": "[network.spec] network Request.headers should define Chrome as user agent header",
+    "platforms": ["darwin", "linux", "win32"],
+    "parameters": ["firefox"],
+    "expectations": ["SKIP"]
+  },
+  {
+    "testIdPattern": "[network.spec] network Request.headers should define Firefox as user agent header",
+    "platforms": ["darwin", "linux", "win32"],
+    "parameters": ["chrome"],
+    "expectations": ["SKIP"]
+  },
+  {
+    "testIdPattern": "[network.spec] network Request.initiator should return the initiator",
+    "platforms": ["darwin", "linux", "win32"],
+    "parameters": ["webDriverBiDi"],
+    "expectations": ["FAIL"]
+  },
+  {
+    "testIdPattern": "[network.spec] network Request.isNavigationRequest should work",
+    "platforms": ["darwin", "linux", "win32"],
+    "parameters": ["webDriverBiDi"],
+    "expectations": ["FAIL"]
+  },
+  {
+    "testIdPattern": "[network.spec] network Request.isNavigationRequest should work with request interception",
+    "platforms": ["darwin", "linux", "win32"],
+    "parameters": ["webDriverBiDi"],
+    "expectations": ["FAIL"]
+  },
+  {
+    "testIdPattern": "[network.spec] network Request.postData should work",
+    "platforms": ["darwin", "linux", "win32"],
+    "parameters": ["webDriverBiDi"],
+    "expectations": ["FAIL"]
+  },
+  {
+    "testIdPattern": "[network.spec] network Response.fromCache should work",
+    "platforms": ["darwin", "linux", "win32"],
+    "parameters": ["webDriverBiDi"],
+    "expectations": ["FAIL"]
+  },
+  {
+    "testIdPattern": "[network.spec] network Response.fromServiceWorker Response.fromServiceWorker",
+    "platforms": ["darwin", "linux", "win32"],
+    "parameters": ["webDriverBiDi"],
+    "expectations": ["FAIL"]
+  },
+  {
+    "testIdPattern": "[network.spec] network Response.fromServiceWorker should return |false| for non-service-worker content",
+    "platforms": ["darwin", "linux", "win32"],
+    "parameters": ["webDriverBiDi"],
+    "expectations": ["FAIL"]
+  },
+  {
+    "testIdPattern": "[network.spec] network Response.timing returns timing information",
+    "platforms": ["darwin", "linux", "win32"],
+    "parameters": ["webDriverBiDi"],
+    "expectations": ["FAIL"]
+  },
   {
     "testIdPattern": "[oopif.spec] *",
     "platforms": ["darwin", "linux", "win32"],
@@ -1176,10 +1248,10 @@
     "expectations": ["FAIL"]
   },
   {
-    "testIdPattern": "[navigation.spec] navigation Page.goto should fail when navigating to bad SSL",
+    "testIdPattern": "[navigation.spec] navigation Page.goto should fail when main resources failed to load",
     "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["cdp", "firefox"],
+    "parameters": ["firefox", "webDriverBiDi"],
-    "expectations": ["FAIL"]
+    "expectations": ["FAIL", "PASS"]
   },
   {
     "testIdPattern": "[navigation.spec] navigation Page.goto should fail when navigating to bad SSL after redirects",
@@ -1211,6 +1283,12 @@
     "parameters": ["cdp", "firefox"],
     "expectations": ["FAIL"]
   },
+  {
+    "testIdPattern": "[navigation.spec] navigation Page.goto should navigate to empty page with domcontentloaded",
+    "platforms": ["darwin", "linux", "win32"],
+    "parameters": ["firefox", "webDriverBiDi"],
+    "expectations": ["FAIL"]
+  },
   {
     "testIdPattern": "[navigation.spec] navigation Page.goto should navigate to empty page with networkidle0",
     "platforms": ["darwin", "linux", "win32"],
@@ -1241,6 +1319,24 @@
     "parameters": ["chrome", "webDriverBiDi"],
     "expectations": ["PASS", "TIMEOUT"]
   },
+  {
+    "testIdPattern": "[navigation.spec] navigation Page.goto should not throw an error for a 404 response with an empty body",
+    "platforms": ["darwin", "linux", "win32"],
+    "parameters": ["firefox", "webDriverBiDi"],
+    "expectations": ["FAIL"]
+  },
+  {
+    "testIdPattern": "[navigation.spec] navigation Page.goto should not throw an error for a 500 response with an empty body",
+    "platforms": ["darwin", "linux", "win32"],
+    "parameters": ["firefox", "webDriverBiDi"],
+    "expectations": ["FAIL"]
+  },
+  {
+    "testIdPattern": "[navigation.spec] navigation Page.goto should return response when page changes its URL after load",
+    "platforms": ["darwin", "linux", "win32"],
+    "parameters": ["firefox", "webDriverBiDi"],
+    "expectations": ["FAIL"]
+  },
   {
     "testIdPattern": "[navigation.spec] navigation Page.goto should send referer",
     "platforms": ["darwin", "linux", "win32"],
@@ -1253,18 +1349,48 @@
     "parameters": ["cdp", "firefox"],
     "expectations": ["SKIP"]
   },
+  {
+    "testIdPattern": "[navigation.spec] navigation Page.goto should work when navigating to 404",
+    "platforms": ["darwin", "linux", "win32"],
+    "parameters": ["firefox", "webDriverBiDi"],
+    "expectations": ["FAIL"]
+  },
+  {
+    "testIdPattern": "[navigation.spec] navigation Page.goto should work when navigating to data url",
+    "platforms": ["darwin", "linux", "win32"],
+    "parameters": ["firefox", "webDriverBiDi"],
+    "expectations": ["FAIL"]
+  },
   {
     "testIdPattern": "[navigation.spec] navigation Page.goto should work when navigating to data url",
     "platforms": ["darwin", "linux", "win32"],
     "parameters": ["cdp", "firefox"],
     "expectations": ["FAIL"]
   },
+  {
+    "testIdPattern": "[navigation.spec] navigation Page.goto should work when navigating to valid url",
+    "platforms": ["darwin", "linux", "win32"],
+    "parameters": ["firefox", "webDriverBiDi"],
+    "expectations": ["FAIL", "TIMEOUT"]
+  },
+  {
+    "testIdPattern": "[navigation.spec] navigation Page.goto should work when page calls history API in beforeunload",
+    "platforms": ["darwin", "linux", "win32"],
+    "parameters": ["firefox", "webDriverBiDi"],
+    "expectations": ["FAIL"]
+  },
   {
     "testIdPattern": "[navigation.spec] navigation Page.goto should work with anchor navigation",
     "platforms": ["linux"],
     "parameters": ["chrome", "headless"],
     "expectations": ["PASS", "TIMEOUT"]
   },
+  {
+    "testIdPattern": "[navigation.spec] navigation Page.goto should work with anchor navigation",
+    "platforms": ["darwin", "linux", "win32"],
+    "parameters": ["chrome", "headless"],
+    "expectations": ["PASS", "TIMEOUT"]
+  },
   {
     "testIdPattern": "[navigation.spec] navigation Page.goto should work with redirects",
     "platforms": ["darwin", "linux", "win32"],
@@ -1421,18 +1547,6 @@
     "parameters": ["cdp", "chrome"],
     "expectations": ["FAIL", "PASS"]
   },
-  {
-    "testIdPattern": "[network.spec] network Request.headers should define Chrome as user agent header",
-    "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["cdp", "firefox"],
-    "expectations": ["SKIP"]
-  },
-  {
-    "testIdPattern": "[network.spec] network Request.headers should define Firefox as user agent header",
-    "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["cdp", "chrome"],
-    "expectations": ["SKIP"]
-  },
   {
     "testIdPattern": "[network.spec] network Request.initiator should return the initiator",
     "platforms": ["darwin", "linux", "win32"],
@@ -2219,6 +2333,12 @@
     "parameters": ["chrome", "headless", "webDriverBiDi"],
     "expectations": ["FAIL"]
   },
+  {
+    "testIdPattern": "[navigation.spec] navigation \"after each\" hook in \"navigation\"",
+    "platforms": ["darwin", "linux", "win32"],
+    "parameters": ["chrome", "headless", "webDriverBiDi"],
+    "expectations": ["FAIL"]
+  },
   {
     "testIdPattern": "[network.spec] network \"after each\" hook for \"Same-origin set-cookie subresource\"",
     "platforms": ["win32"],

test/src/evaluation.spec.ts

@@ -480,7 +480,7 @@ describe('Evaluation specs', function () {
       });
       expect(a.length).toBe(100 * 1024 * 1024);
     });
-    it('should throw error with detailed information on exception inside promise ', async () => {
+    it('should throw error with detailed information on exception inside promise', async () => {
       const {page} = getTestState();
 
       let error!: Error;

test/src/mocha-utils.ts

@@ -355,18 +355,18 @@ export const launch = async (
       ...defaultBrowserOptions,
       ...options,
     });
-    browserCleanups.push(async () => {
+    browserCleanups.push(() => {
-      browser.close();
+      return browser.close();
     });
 
     const context = await browser.createIncognitoBrowserContext();
-    browserCleanups.push(async () => {
+    browserCleanups.push(() => {
-      context.close();
+      return context.close();
     });
 
     const page = await context.newPage();
-    browserCleanups.push(async () => {
+    browserCleanups.push(() => {
-      page.close();
+      return page.close();
     });
 
     return {

test/src/navigation.spec.ts

@@ -65,8 +65,8 @@ describe('navigation', function () {
     it('should return response when page changes its URL after load', async () => {
       const {page, server} = getTestState();
 
-      const response = (await page.goto(server.PREFIX + '/historyapi.html'))!;
+      const response = await page.goto(server.PREFIX + '/historyapi.html');
-      expect(response.status()).toBe(200);
+      expect(response!.status()).toBe(200);
     });
     it('should work with subframes return 204', async () => {
       const {page, server} = getTestState();

test/src/network.spec.ts

@@ -560,10 +560,11 @@ describe('network', function () {
       });
       await page.goto(server.EMPTY_PAGE);
       expect(requests).toHaveLength(1);
-      expect(requests[0]!.url()).toBe(server.EMPTY_PAGE);
+      const request = requests[0]!;
-      expect(requests[0]!.response()).toBeTruthy();
+      expect(request.url()).toBe(server.EMPTY_PAGE);
-      expect(requests[0]!.frame() === page.mainFrame()).toBe(true);
+      expect(request.response()).toBeTruthy();
-      expect(requests[0]!.frame()!.url()).toBe(server.EMPTY_PAGE);
+      expect(request.frame() === page.mainFrame()).toBe(true);
+      expect(request.frame()!.url()).toBe(server.EMPTY_PAGE);
     });
     it('should fire events in proper order', async () => {
       const {page, server} = getTestState();
@@ -655,12 +656,11 @@ describe('network', function () {
     it('should work when navigating to image', async () => {
       const {page, server} = getTestState();
 
-      const requests: HTTPRequest[] = [];
+      const [request] = await Promise.all([
-      page.on('request', request => {
+        waitEvent<HTTPRequest>(page, 'request'),
-        return requests.push(request);
+        page.goto(server.PREFIX + '/pptr.png'),
-      });
+      ]);
-      await page.goto(server.PREFIX + '/pptr.png');
+      expect(request.isNavigationRequest()).toBe(true);
-      expect(requests[0]!.isNavigationRequest()).toBe(true);
     });
   });
 
@@ -813,14 +813,15 @@ describe('network', function () {
         res.end('hello world');
       });
 
-      const responsePromise = waitEvent<HTTPResponse>(page, 'response');
+      const [response] = await Promise.all([
-      page.evaluate(() => {
+        waitEvent<HTTPResponse>(page, 'response'),
-        const xhr = new XMLHttpRequest();
+        page.evaluate(() => {
-        xhr.open('GET', '/foo');
+          const xhr = new XMLHttpRequest();
-        xhr.send();
+          xhr.open('GET', '/foo');
-      });
+          xhr.send();
-      const subresourceResponse = await responsePromise;
+        }),
-      expect(subresourceResponse.headers()['set-cookie']).toBe(setCookieString);
+      ]);
+      expect(response.headers()['set-cookie']).toBe(setCookieString);
     });
 
     it('Cross-origin set-cookie', async () => {

test/src/page.spec.ts

@@ -124,7 +124,7 @@ describe('Page', function () {
     it('should fire when expected', async () => {
       const {page} = getTestState();
 
-      await Promise.all([page.goto('about:blank'), waitEvent(page, 'load')]);
+      await Promise.all([waitEvent(page, 'load'), page.goto('about:blank')]);
     });
   });
 

tools/mochaRunner/src/utils.ts

@@ -76,14 +76,16 @@ export function printSuggestions(
         return item.expectation;
       })
     );
-    console.log(
+    if (action !== 'remove') {
-      'The recommendations are based on the following applied expectaions:'
+      console.log(
-    );
+        'The recommendations are based on the following applied expectations:'
-    prettyPrintJSON(
+      );
-      toPrint.map(item => {
+      prettyPrintJSON(
-        return item.basedOn;
+        toPrint.map(item => {
-      })
+          return item.basedOn;
-    );
+        })
+      );
+    }
   }
 }
 

tsconfig.base.json

@@ -27,6 +27,7 @@
     "strictNullChecks": true,
     "strictPropertyInitialization": true,
     "target": "ES2019",
-    "useUnknownInCatchVariables": true
+    "useUnknownInCatchVariables": true,
+    "skipLibCheck": true
   }
 }