feat(page): Adding support for referrerPolicy in page.goto (#9561)

Issue: #9394

**What kind of change does this PR introduce?**

Feature - Added support of the `referrerPolicy` parameter (from CDP's
`Page.navigate`) to Puppeteer's `page.goto`.

**Did you add tests for your changes?**

No, currently it has no meaning that isn't browser implementation based,
which might get broken in the future. If there are suggestions to tests,
please let me know so I'll add them.

**If relevant, did you update the documentation?**
Yes, the documentation of the `goto` method has been updated.

**Summary**
I wanted to contribute to this project, which I used for testing on our
environment and saw issue #9394, so I decided to resolve it :)

**Does this PR introduce a breaking change?**
No. I added the `referrerPolicy` as an optional parameter, which will
use the HTTP header `Referrer-Policy` if not provided (much like the
`referer` parameter) so it will not interfere.

Co-authored-by: Ophir Back <ophir.back@broadcom.com>

docs/api/puppeteer.frame.goto.md

@@ -14,6 +14,7 @@ class Frame {
     url: string,
     options?: {
       referer?: string;
+      referrerPolicy?: string;
       timeout?: number;
       waitUntil?: PuppeteerLifeCycleEvent | PuppeteerLifeCycleEvent[];
     }
@@ -24,9 +25,9 @@ class Frame {
 ## Parameters
 
 | Parameter | Type                                                                                                                                                                                                                     | Description                                                                                                                                                                                                                                 |
-| --------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| --------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
 | url       | string                                                                                                                                                                                                                   | the URL to navigate the frame to. This should include the scheme, e.g. <code>https://</code>.                                                                                                                                               |
-| options   | { referer?: string; timeout?: number; waitUntil?: [PuppeteerLifeCycleEvent](./puppeteer.puppeteerlifecycleevent.md) \| [PuppeteerLifeCycleEvent](./puppeteer.puppeteerlifecycleevent.md)\[\]; } | <i>(Optional)</i> navigation options. <code>waitUntil</code> is useful to define when the navigation should be considered successful - see the docs for [PuppeteerLifeCycleEvent](./puppeteer.puppeteerlifecycleevent.md) for more details. |
+| options   | { referer?: string; referrerPolicy?: string; timeout?: number; waitUntil?: [PuppeteerLifeCycleEvent](./puppeteer.puppeteerlifecycleevent.md) \| [PuppeteerLifeCycleEvent](./puppeteer.puppeteerlifecycleevent.md)\[\]; } | <i>(Optional)</i> navigation options. <code>waitUntil</code> is useful to define when the navigation should be considered successful - see the docs for [PuppeteerLifeCycleEvent](./puppeteer.puppeteerlifecycleevent.md) for more details. |
 
 **Returns:**
 

docs/api/puppeteer.page.goto.md

@@ -12,6 +12,7 @@ class Page {
     url: string,
     options?: WaitForOptions & {
       referer?: string;
+      referrerPolicy?: string;
     }
   ): Promise<HTTPResponse | null>;
 }
@@ -20,9 +21,9 @@ class Page {
 ## Parameters
 
 | Parameter | Type                                                                                                 | Description                                                                        |
-| --------- | --------------------------------------------------------------------------- | ---------------------------------------------------------------------------------- |
+| --------- | ---------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------- |
 | url       | string                                                                                               | URL to navigate page to. The URL should include scheme, e.g. <code>https://</code> |
-| options   | [WaitForOptions](./puppeteer.waitforoptions.md) &amp; { referer?: string; } | <i>(Optional)</i> Navigation Parameter                                             |
+| options   | [WaitForOptions](./puppeteer.waitforoptions.md) &amp; { referer?: string; referrerPolicy?: string; } | <i>(Optional)</i> Navigation Parameter                                             |
 
 **Returns:**
 
@@ -38,7 +39,7 @@ The argument `options` might have the following properties:
 
 - `waitUntil`:When to consider navigation succeeded, defaults to `load`. Given an array of event strings, navigation is considered to be successful after all events have been fired. Events can be either:<br/> - `load` : consider navigation to be finished when the load event is fired.<br/> - `domcontentloaded` : consider navigation to be finished when the DOMContentLoaded event is fired.<br/> - `networkidle0` : consider navigation to be finished when there are no more than 0 network connections for at least `500` ms.<br/> - `networkidle2` : consider navigation to be finished when there are no more than 2 network connections for at least `500` ms.
 
-- `referer` : Referer header value. If provided it will take preference over the referer header value set by [page.setExtraHTTPHeaders()](./puppeteer.page.setextrahttpheaders.md).
+- `referer` : Referer header value. If provided it will take preference over the referer header value set by [page.setExtraHTTPHeaders()](./puppeteer.page.setextrahttpheaders.md).<br/> - `referrerPolicy` : ReferrerPolicy. If provided it will take preference over the referer-policy header value set by [page.setExtraHTTPHeaders()](./puppeteer.page.setextrahttpheaders.md).
 
 `page.goto` will throw an error if:
 

packages/puppeteer-core/src/api/Page.ts

@@ -1381,6 +1381,9 @@ export class Page extends EventEmitter {
    *
    * - `referer` : Referer header value. If provided it will take preference
    *   over the referer header value set by
+   *   {@link Page.setExtraHTTPHeaders |page.setExtraHTTPHeaders()}.<br/>
+   * - `referrerPolicy` : ReferrerPolicy. If provided it will take preference
+   *   over the referer-policy header value set by
    *   {@link Page.setExtraHTTPHeaders |page.setExtraHTTPHeaders()}.
    *
    * `page.goto` will throw an error if:
@@ -1408,7 +1411,7 @@ export class Page extends EventEmitter {
    */
   async goto(
     url: string,
-    options?: WaitForOptions & {referer?: string}
+    options?: WaitForOptions & {referer?: string; referrerPolicy?: string}
   ): Promise<HTTPResponse | null>;
   async goto(): Promise<HTTPResponse | null> {
     throw new Error('Not implemented');

packages/puppeteer-core/src/common/Frame.ts

@@ -289,12 +289,16 @@ export class Frame {
     url: string,
     options: {
       referer?: string;
+      referrerPolicy?: string;
       timeout?: number;
       waitUntil?: PuppeteerLifeCycleEvent | PuppeteerLifeCycleEvent[];
     } = {}
   ): Promise<HTTPResponse | null> {
     const {
       referer = this._frameManager.networkManager.extraHTTPHeaders()['referer'],
+      referrerPolicy = this._frameManager.networkManager.extraHTTPHeaders()[
+        'referer-policy'
+      ],
       waitUntil = ['load'],
       timeout = this._frameManager.timeoutSettings.navigationTimeout(),
     } = options;
@@ -307,7 +311,13 @@ export class Frame {
       timeout
     );
     let error = await Promise.race([
-      navigate(this.#client, url, referer, this._id),
+      navigate(
+        this.#client,
+        url,
+        referer,
+        referrerPolicy as Protocol.Page.ReferrerPolicy,
+        this._id
+      ),
       watcher.timeoutOrTerminationPromise(),
     ]);
     if (!error) {
@@ -332,6 +342,7 @@ export class Frame {
       client: CDPSession,
       url: string,
       referrer: string | undefined,
+      referrerPolicy: Protocol.Page.ReferrerPolicy | undefined,
       frameId: string
     ): Promise<Error | null> {
       try {
@@ -339,6 +350,7 @@ export class Frame {
           url,
           referrer,
           frameId,
+          referrerPolicy,
         });
         ensureNewDocumentNavigation = !!response.loaderId;
         return response.errorText

packages/puppeteer-core/src/common/Page.ts

@@ -893,7 +893,7 @@ export class CDPPage extends Page {
 
   override async goto(
     url: string,
-    options: WaitForOptions & {referer?: string} = {}
+    options: WaitForOptions & {referer?: string; referrerPolicy?: string} = {}
   ): Promise<HTTPResponse | null> {
     return await this.#frameManager.mainFrame().goto(url, options);
   }

test/src/navigation.spec.ts

@@ -494,6 +494,20 @@ describe('navigation', function () {
       // Make sure subresources do not inherit referer.
       expect(request2.headers['referer']).toBe(server.PREFIX + '/grid.html');
     });
+
+    it('should send referer policy', async () => {
+      const {page, server} = getTestState();
+
+      const [request1, request2] = await Promise.all([
+        server.waitForRequest('/grid.html'),
+        server.waitForRequest('/digits/1.png'),
+        page.goto(server.PREFIX + '/grid.html', {
+          referrerPolicy: 'no-referer',
+        }),
+      ]);
+      expect(request1.headers['referer']).toBeUndefined();
+      expect(request2.headers['referer']).toBe(server.PREFIX + '/grid.html');
+    });
   });
 
   describe('Page.waitForNavigation', function () {