chore: add BiDi integration for Chromium (#9410)

This PR adds experimental support for WebDriver BiDi by making use of
chromium-bidi to implement the BiDi protocol. The tests are disabled on
Windows due to flakiness (filed
https://github.com/GoogleChromeLabs/chromium-bidi/issues/361).

.github/workflows/ci.yml

@@ -132,6 +132,10 @@ jobs:
           - chrome-headless
           - chrome-headful
           - chrome-new-headless
+          - chrome-bidi
+        exclude:
+          - os: windows-latest
+            suite: chrome-bidi
     steps:
       - name: Check out repository
         uses: actions/checkout@v3.0.2

docs/api/puppeteer.eventtype.md

@@ -7,5 +7,5 @@ sidebar_label: EventType
 #### Signature:
 
 ```typescript
-declare type EventType = string | symbol;
+export type EventType = string | symbol;
 ```

docs/api/puppeteer.handler.md

@@ -7,5 +7,5 @@ sidebar_label: Handler
 #### Signature:
 
 ```typescript
-declare type Handler<T = unknown> = (event: T) => void;
+export type Handler<T = unknown> = (event: T) => void;
 ```

packages/puppeteer-core/src/common/EventEmitter.ts

@@ -14,17 +14,16 @@
  * limitations under the License.
  */
 
-import mitt, {
+import mitt, {Emitter, EventHandlerMap} from '../../third_party/mitt/index.js';
-  Emitter,
-  EventType,
-  EventHandlerMap,
-  Handler,
-} from '../../third_party/mitt/index.js';
 
 /**
  * @public
  */
-export {EventType, Handler};
+export type EventType = string | symbol;
+/**
+ * @public
+ */
+export type Handler<T = unknown> = (event: T) => void;
 
 /**
  * @public

packages/puppeteer-core/src/common/bidi/BidiOverCDP.ts

@@ -0,0 +1,167 @@
+import {CDPSession, Connection as CDPPPtrConnection} from '../Connection.js';
+import {Connection as BidiPPtrConnection} from './Connection.js';
+import {Bidi, BidiMapper} from '../../../third_party/chromium-bidi/index.js';
+import type {ProtocolMapping} from 'devtools-protocol/types/protocol-mapping.js';
+import {Handler} from '../EventEmitter.js';
+
+type CdpEvents = {
+  [Property in keyof ProtocolMapping.Events]: ProtocolMapping.Events[Property][0];
+};
+
+/**
+ * @internal
+ */
+export async function connectBidiOverCDP(
+  cdp: CDPPPtrConnection
+): Promise<BidiPPtrConnection> {
+  const transportBiDi = new NoOpTransport();
+  const cdpConnectionAdapter = new CDPConnectionAdapter(cdp);
+  const pptrTransport = {
+    send(message: string): void {
+      // Forwards a BiDi command sent by Puppeteer to the input of the BidiServer.
+      transportBiDi.emitMessage(JSON.parse(message));
+    },
+    close(): void {
+      bidiServer.close();
+      cdpConnectionAdapter.close();
+    },
+    onmessage(_message: string): void {
+      // The method is overriden by the Connection.
+    },
+  };
+  transportBiDi.on('bidiResponse', (message: object) => {
+    // Forwards a BiDi event sent by BidiServer to Puppeteer.
+    pptrTransport.onmessage(JSON.stringify(message));
+  });
+  const pptrBiDiConnection = new BidiPPtrConnection(pptrTransport);
+  const bidiServer = await BidiMapper.BidiServer.createAndStart(
+    transportBiDi,
+    cdpConnectionAdapter,
+    ''
+  );
+  return pptrBiDiConnection;
+}
+
+/**
+ * Manages CDPSessions for BidiServer.
+ * @internal
+ */
+class CDPConnectionAdapter {
+  #cdp: CDPPPtrConnection;
+  #adapters = new Map<CDPSession, CDPClientAdapter<CDPSession>>();
+  #browser: CDPClientAdapter<CDPPPtrConnection>;
+
+  constructor(cdp: CDPPPtrConnection) {
+    this.#cdp = cdp;
+    this.#browser = new CDPClientAdapter(cdp);
+  }
+
+  browserClient(): CDPClientAdapter<CDPPPtrConnection> {
+    return this.#browser;
+  }
+
+  getCdpClient(id: string) {
+    const session = this.#cdp.session(id);
+    if (!session) {
+      throw new Error('Unknonw CDP session with id' + id);
+    }
+    if (!this.#adapters.has(session)) {
+      const adapter = new CDPClientAdapter(session);
+      this.#adapters.set(session, adapter);
+      return adapter;
+    }
+    return this.#adapters.get(session)!;
+  }
+
+  close() {
+    this.#browser.close();
+    for (const adapter of this.#adapters.values()) {
+      adapter.close();
+    }
+  }
+}
+
+/**
+ * Wrapper on top of CDPSession/CDPConnection to satisify CDP interface that
+ * BidiServer needs.
+ *
+ * @internal
+ */
+class CDPClientAdapter<
+  T extends Pick<CDPPPtrConnection, 'send' | 'on' | 'off'>
+> extends BidiMapper.EventEmitter<CdpEvents> {
+  #closed = false;
+  #client: T;
+
+  constructor(client: T) {
+    super();
+    this.#client = client;
+    this.#client.on('*', this.#forwardMessage as Handler<any>);
+  }
+
+  #forwardMessage = (
+    method: keyof ProtocolMapping.Events,
+    event: ProtocolMapping.Events[keyof ProtocolMapping.Events]
+  ) => {
+    this.emit(method, event);
+  };
+
+  async sendCommand<T extends keyof ProtocolMapping.Commands>(
+    method: T,
+    ...params: ProtocolMapping.Commands[T]['paramsType']
+  ): Promise<ProtocolMapping.Commands[T]['returnType']> {
+    if (this.#closed) {
+      return;
+    }
+    try {
+      return await this.#client.send(method, ...params);
+    } catch (err) {
+      if (this.#closed) {
+        return;
+      }
+      throw err;
+    }
+  }
+
+  close() {
+    this.#client.off('*', this.#forwardMessage as Handler<any>);
+    this.#closed = true;
+  }
+}
+
+/**
+ * This transport is given to the BiDi server instance and allows Puppeteer
+ * to send and receive commands to the BiDiServer.
+ * @internal
+ */
+class NoOpTransport
+  extends BidiMapper.EventEmitter<any>
+  implements BidiMapper.BidiTransport
+{
+  #onMessage: (message: Bidi.Message.RawCommandRequest) => Promise<void> =
+    async (_m: Bidi.Message.RawCommandRequest): Promise<void> => {
+      return;
+    };
+
+  emitMessage(message: Bidi.Message.RawCommandRequest) {
+    this.#onMessage(message);
+  }
+
+  setOnMessage(
+    onMessage: (message: Bidi.Message.RawCommandRequest) => Promise<void>
+  ): void {
+    this.#onMessage = onMessage;
+  }
+
+  async sendMessage(message: Bidi.Message.OutgoingMessage): Promise<void> {
+    this.emit('bidiResponse', message);
+  }
+
+  close() {
+    this.#onMessage = async (
+      _m: Bidi.Message.RawCommandRequest
+    ): Promise<void> => {
+      return;
+    };
+  }
+}

packages/puppeteer-core/src/common/bidi/Browser.ts

@@ -33,7 +33,9 @@ export class Browser extends BrowserBase {
    */
   static async create(opts: Options): Promise<Browser> {
     // TODO: await until the connection is established.
-    (await opts.connection.send('session.new', {})) as {sessionId: string};
+    try {
+      (await opts.connection.send('session.new', {})) as {sessionId: string};
+    } catch {}
     return new Browser(opts);
   }
 
@@ -52,8 +54,8 @@ export class Browser extends BrowserBase {
   }
 
   override async close(): Promise<void> {
-    await this.#closeCallback?.call(null);
     this.#connection.dispose();
+    await this.#closeCallback?.call(null);
   }
 
   override isConnected(): boolean {

packages/puppeteer-core/src/common/bidi/bidi.ts

@@ -18,3 +18,4 @@ export * from './Browser.js';
 export * from './BrowserContext.js';
 export * from './Page.js';
 export * from './Connection.js';
+export * from './BidiOverCDP.js';

packages/puppeteer-core/src/node/ChromeLauncher.ts

@@ -2,9 +2,10 @@ import {accessSync} from 'fs';
 import {mkdtemp} from 'fs/promises';
 import os from 'os';
 import path from 'path';
-import {CDPBrowser} from '../common/Browser.js';
+import {Browser} from '../api/Browser.js';
 import {assert} from '../util/assert.js';
 import {BrowserRunner} from './BrowserRunner.js';
+import {CDPBrowser} from '../common/Browser.js';
 import {
   BrowserLaunchArgumentOptions,
   ChromeReleaseChannel,
@@ -23,7 +24,7 @@ export class ChromeLauncher extends ProductLauncher {
 
   override async launch(
     options: PuppeteerNodeLaunchOptions = {}
-  ): Promise<CDPBrowser> {
+  ): Promise<Browser> {
     const {
       ignoreDefaultArgs = false,
       args = [],
@@ -41,6 +42,7 @@ export class ChromeLauncher extends ProductLauncher {
       timeout = 30000,
       waitForInitialPage = true,
       debuggingPort,
+      protocol,
     } = options;
 
     const chromeArguments = [];
@@ -124,6 +126,24 @@ export class ChromeLauncher extends ProductLauncher {
         slowMo,
         preferredRevision: this.puppeteer.browserRevision,
       });
+
+      if (protocol === 'webDriverBiDi') {
+        try {
+          const BiDi = await import('../common/bidi/bidi.js');
+          const bidiConnection = await BiDi.connectBidiOverCDP(connection);
+          browser = await BiDi.Browser.create({
+            connection: bidiConnection,
+            closeCallback: runner.close.bind(runner),
+            process: runner.proc,
+          });
+        } catch (error) {
+          runner.kill();
+          throw error;
+        }
+
+        return browser;
+      }
+
       browser = await CDPBrowser._create(
         this.product,
         connection,

packages/puppeteer-core/third_party/chromium-bidi/index.ts

@@ -0,0 +1,18 @@
+/**
+ * Copyright 2022 Google Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export * as BidiMapper from 'chromium-bidi/lib/cjs/bidiMapper/bidiMapper.js';
+export * as Bidi from 'chromium-bidi/lib/cjs/protocol/protocol.js';

test/TestExpectations.json

@@ -2066,49 +2066,49 @@
   {
     "testIdPattern": "",
     "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["firefox", "webDriverBiDi"],
+    "parameters": ["webDriverBiDi"],
     "expectations": ["SKIP", "TIMEOUT"]
   },
   {
     "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.launch can launch and close the browser",
     "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["firefox", "webDriverBiDi"],
+    "parameters": ["webDriverBiDi"],
     "expectations": ["PASS"]
   },
   {
     "testIdPattern": "[Connection.spec] WebDriver BiDi",
     "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["firefox", "webDriverBiDi"],
+    "parameters": ["webDriverBiDi"],
     "expectations": ["PASS"]
   },
   {
     "testIdPattern": "[evaluation.spec] Evaluation specs Page.evaluate should work",
     "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["firefox", "webDriverBiDi"],
+    "parameters": ["webDriverBiDi"],
     "expectations": ["PASS"]
   },
   {
     "testIdPattern": "[evaluation.spec] Evaluation specs Page.evaluate should work with function shorthands",
     "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["firefox", "webDriverBiDi"],
+    "parameters": ["webDriverBiDi"],
     "expectations": ["SKIP"]
   },
   {
     "testIdPattern": "[evaluation.spec] Evaluation specs Page.evaluate should work with unicode chars",
     "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["firefox", "webDriverBiDi"],
+    "parameters": ["webDriverBiDi"],
     "expectations": ["SKIP"]
   },
   {
     "testIdPattern": "[evaluation.spec] Evaluation specs Page.evaluate should work right after framenavigated",
     "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["firefox", "webDriverBiDi"],
+    "parameters": ["webDriverBiDi"],
     "expectations": ["SKIP"]
   },
   {
     "testIdPattern": "[evaluation.spec] Evaluation specs Page.evaluate should work from-inside an exposed function",
     "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["firefox", "webDriverBiDi"],
+    "parameters": ["webDriverBiDi"],
     "expectations": ["SKIP"]
   }
 ]

test/TestSuites.json

@@ -35,6 +35,12 @@
       "platforms": ["linux"],
       "parameters": ["firefox", "headless", "webDriverBiDi"],
       "expectedLineCoverage": 56
+    },
+    {
+      "id": "chrome-bidi",
+      "platforms": ["linux"],
+      "parameters": ["chrome", "headless", "webDriverBiDi"],
+      "expectedLineCoverage": 56
     }
   ],
   "parameterDefinitons": {