chore(webdriver): support page.setExtraHTTPHeaders (#12269)

Will create a separate PR that changes the docs and marked as `feat`.

---------

Co-authored-by: Alex Rudenko <OrKoN@users.noreply.github.com>

packages/puppeteer-core/src/bidi/HTTPRequest.ts

@@ -75,6 +75,17 @@ export class BidiHTTPRequest extends HTTPRequest {
     this.#request.on('authenticate', this.#handleAuthentication);
 
     this.#frame.page().trustedEmitter.emit(PageEvent.Request, this);
+
+    if (Object.keys(this.#extraHTTPHeaders).length) {
+      this.interception.handlers.push(async () => {
+        await this.continue(
+          {
+            headers: this.headers(),
+          },
+          0
+        );
+      });
+    }
   }
 
   override url(): string {
@@ -101,12 +112,19 @@ export class BidiHTTPRequest extends HTTPRequest {
     throw new UnsupportedOperation();
   }
 
+  get #extraHTTPHeaders(): Record<string, string> {
+    return this.#frame?.page()._extraHTTPHeaders ?? {};
+  }
+
   override headers(): Record<string, string> {
     const headers: Record<string, string> = {};
     for (const header of this.#request.headers) {
       headers[header.name.toLowerCase()] = header.value.value;
     }
-    return headers;
+    return {
+      ...headers,
+      ...this.#extraHTTPHeaders,
+    };
   }
 
   override response(): BidiHTTPResponse | null {
@@ -145,6 +163,21 @@ export class BidiHTTPRequest extends HTTPRequest {
     return this.#frame;
   }
 
+  override async continue(
+    overrides?: ContinueRequestOverrides,
+    priority?: number | undefined
+  ): Promise<void> {
+    return await super.continue(
+      {
+        headers: Object.keys(this.#extraHTTPHeaders).length
+          ? this.headers()
+          : undefined,
+        ...overrides,
+      },
+      priority
+    );
+  }
+
   override async _continue(
     overrides: ContinueRequestOverrides = {}
   ): Promise<void> {

packages/puppeteer-core/src/bidi/Page.ts

@@ -38,7 +38,12 @@ import {UnsupportedOperation} from '../common/Errors.js';
 import {EventEmitter} from '../common/EventEmitter.js';
 import type {PDFOptions} from '../common/PDFOptions.js';
 import type {Awaitable} from '../common/types.js';
-import {evaluationString, parsePDFOptions, timeout} from '../common/util.js';
+import {
+  evaluationString,
+  isString,
+  parsePDFOptions,
+  timeout,
+} from '../common/util.js';
 import type {Viewport} from '../common/Viewport.js';
 import {assert} from '../util/assert.js';
 import {bubble} from '../util/decorators.js';
@@ -520,6 +525,31 @@ export class BidiPage extends Page {
     );
   }
 
+  /**
+   * @internal
+   */
+  _extraHTTPHeaders: Record<string, string> = {};
+  #extraHeadersInterception?: string;
+  override async setExtraHTTPHeaders(
+    headers: Record<string, string>
+  ): Promise<void> {
+    const extraHTTPHeaders: Record<string, string> = {};
+    for (const [key, value] of Object.entries(headers)) {
+      assert(
+        isString(value),
+        `Expected value of header "${key}" to be String, but "${typeof value}" is found.`
+      );
+      extraHTTPHeaders[key.toLowerCase()] = value;
+    }
+    this._extraHTTPHeaders = extraHTTPHeaders;
+
+    this.#extraHeadersInterception = await this.#toggleInterception(
+      [Bidi.Network.InterceptPhase.BeforeRequestSent],
+      this.#extraHeadersInterception,
+      Boolean(Object.keys(this._extraHTTPHeaders).length)
+    );
+  }
+
   /**
    * @internal
    */
@@ -663,10 +693,6 @@ export class BidiPage extends Page {
     await this.#frame.removeExposedFunction(name);
   }
 
-  override setExtraHTTPHeaders(): never {
-    throw new UnsupportedOperation();
-  }
-
   override metrics(): never {
     throw new UnsupportedOperation();
   }

test/TestExpectations.json

@@ -97,13 +97,6 @@
     "expectations": ["SKIP"],
     "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)"
   },
-  {
-    "testIdPattern": "[network.spec] network Page.setExtraHTTPHeaders *",
-    "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["webDriverBiDi"],
-    "expectations": ["FAIL"],
-    "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)"
-  },
   {
     "testIdPattern": "[network.spec] network Request.isNavigationRequest *",
     "platforms": ["darwin", "linux", "win32"],
@@ -556,6 +549,13 @@
     "parameters": ["webDriverBiDi"],
     "expectations": ["PASS"]
   },
+  {
+    "testIdPattern": "[network.spec] network Page.setExtraHTTPHeaders *",
+    "platforms": ["darwin", "linux", "win32"],
+    "parameters": ["firefox", "webDriverBiDi"],
+    "expectations": ["SKIP"],
+    "comment": "Firefox does not support headers override"
+  },
   {
     "testIdPattern": "[network.spec] network Request.initiator should return the initiator",
     "platforms": ["darwin", "linux", "win32"],
@@ -840,27 +840,6 @@
     "expectations": ["FAIL"],
     "comment": "BiDi spec and WPT require expect the Hash"
   },
-  {
-    "testIdPattern": "[requestinterception.spec] request interception Page.setRequestInterception should send referer",
-    "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["webDriverBiDi"],
-    "expectations": ["FAIL"],
-    "comment": "`setExtraHTTPHeaders` not implemented"
-  },
-  {
-    "testIdPattern": "[requestinterception.spec] request interception Page.setRequestInterception should show custom HTTP headers",
-    "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["webDriverBiDi"],
-    "expectations": ["FAIL"],
-    "comment": "`setExtraHTTPHeaders` not implemented"
-  },
-  {
-    "testIdPattern": "[requestinterception.spec] request interception Page.setRequestInterception should work with custom referer headers",
-    "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["webDriverBiDi"],
-    "expectations": ["FAIL"],
-    "comment": "`setExtraHTTPHeaders` not implemented"
-  },
   {
     "testIdPattern": "[requestinterception.spec] request interception Page.setRequestInterception should work with redirects",
     "platforms": ["darwin", "linux", "win32"],
@@ -3457,14 +3436,14 @@
     "platforms": ["darwin", "linux", "win32"],
     "parameters": ["firefox", "webDriverBiDi"],
     "expectations": ["SKIP"],
-    "comment": "TODO: Needs support for Page.setExtraHTTPHeaders"
+    "comment": "Firefox does not support headers override"
   },
   {
     "testIdPattern": "[requestinterception-experimental.spec] cooperative request interception Page.setRequestInterception should show custom HTTP headers",
     "platforms": ["darwin", "linux", "win32"],
     "parameters": ["firefox", "webDriverBiDi"],
     "expectations": ["SKIP"],
-    "comment": "TODO: Needs support for Page.setExtraHTTPHeaders"
+    "comment": "Firefox does not support headers override"
   },
   {
     "testIdPattern": "[requestinterception-experimental.spec] cooperative request interception Page.setRequestInterception should work when header manipulation headers with redirect",
@@ -3478,7 +3457,7 @@
     "platforms": ["darwin", "linux", "win32"],
     "parameters": ["firefox", "webDriverBiDi"],
     "expectations": ["SKIP"],
-    "comment": "TODO: Needs support for Page.setExtraHTTPHeaders"
+    "comment": "Firefox does not support headers override"
   },
   {
     "testIdPattern": "[requestinterception-experimental.spec] cooperative request interception Page.setRequestInterception should work with encoded server - 2",
@@ -3492,7 +3471,7 @@
     "platforms": ["darwin", "linux", "win32"],
     "parameters": ["firefox", "webDriverBiDi"],
     "expectations": ["FAIL"],
-    "comment": "TODO: Needs investigation"
+    "comment": "TODO: Needs investigation, it looks like Firefox lets the request go also to the server"
   },
   {
     "testIdPattern": "[requestinterception-experimental.spec] cooperative request interception Page.setRequestInterception should work with file URLs",
@@ -3625,19 +3604,33 @@
     "expectations": ["SKIP"],
     "comment": "TODO: Needs support for enabling cache in BiDi without CDP https://github.com/w3c/webdriver-bidi/issues/582"
   },
+  {
+    "testIdPattern": "[requestinterception.spec] request interception Page.setRequestInterception should send referer",
+    "platforms": ["darwin", "linux", "win32"],
+    "parameters": ["firefox", "webDriverBiDi"],
+    "expectations": ["FAIL"],
+    "comment": "Firefox does not support headers override"
+  },
   {
     "testIdPattern": "[requestinterception.spec] request interception Page.setRequestInterception should send referer",
     "platforms": ["darwin", "linux", "win32"],
     "parameters": ["firefox", "webDriverBiDi"],
     "expectations": ["SKIP"],
-    "comment": "TODO: Needs support for Page.setExtraHTTPHeaders"
+    "comment": "Firefox does not support headers override"
+  },
+  {
+    "testIdPattern": "[requestinterception.spec] request interception Page.setRequestInterception should show custom HTTP headers",
+    "platforms": ["darwin", "linux", "win32"],
+    "parameters": ["firefox", "webDriverBiDi"],
+    "expectations": ["FAIL"],
+    "comment": "Firefox does not support headers override"
   },
   {
     "testIdPattern": "[requestinterception.spec] request interception Page.setRequestInterception should show custom HTTP headers",
     "platforms": ["darwin", "linux", "win32"],
     "parameters": ["firefox", "webDriverBiDi"],
     "expectations": ["SKIP"],
-    "comment": "TODO: Needs support for Page.setExtraHTTPHeaders"
+    "comment": "Firefox does not support headers override"
   },
   {
     "testIdPattern": "[requestinterception.spec] request interception Page.setRequestInterception should work when header manipulation headers with redirect",
@@ -3646,12 +3639,19 @@
     "expectations": ["SKIP"],
     "comment": "TODO: Needs investigation, on Firefox the test is passing even if headers are not actually modified"
   },
+  {
+    "testIdPattern": "[requestinterception.spec] request interception Page.setRequestInterception should work with custom referer headers",
+    "platforms": ["darwin", "linux", "win32"],
+    "parameters": ["firefox", "webDriverBiDi"],
+    "expectations": ["FAIL"],
+    "comment": "Firefox does not support headers override"
+  },
   {
     "testIdPattern": "[requestinterception.spec] request interception Page.setRequestInterception should work with custom referer headers",
     "platforms": ["darwin", "linux", "win32"],
     "parameters": ["firefox", "webDriverBiDi"],
     "expectations": ["SKIP"],
-    "comment": "TODO: Needs support for Page.setExtraHTTPHeaders"
+    "comment": "Firefox does not support headers override"
   },
   {
     "testIdPattern": "[requestinterception.spec] request interception Page.setRequestInterception should work with encoded server - 2",
@@ -4133,27 +4133,6 @@
     "expectations": ["FAIL"],
     "comment": "`request.postData()` has no eqivalent in BiDi spec"
   },
-  {
-    "testIdPattern": "[requestinterception-experimental.spec] cooperative request interception Page.setRequestInterception should send referer",
-    "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["chrome", "headless", "webDriverBiDi"],
-    "expectations": ["FAIL"],
-    "comment": "`setExtraHTTPHeaders` not implemented"
-  },
-  {
-    "testIdPattern": "[requestinterception-experimental.spec] cooperative request interception Page.setRequestInterception should show custom HTTP headers",
-    "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["chrome", "headless", "webDriverBiDi"],
-    "expectations": ["FAIL"],
-    "comment": "`setExtraHTTPHeaders` not implemented"
-  },
-  {
-    "testIdPattern": "[requestinterception-experimental.spec] cooperative request interception Page.setRequestInterception should work with custom referer headers",
-    "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["chrome", "headless", "webDriverBiDi"],
-    "expectations": ["FAIL"],
-    "comment": "`setExtraHTTPHeaders` not implemented"
-  },
   {
     "testIdPattern": "[requestinterception-experimental.spec] cooperative request interception Page.setRequestInterception should work with redirects",
     "platforms": ["darwin", "linux", "win32"],