From e3d69ec554beeac37bd206a21921d2fed3cb968c Mon Sep 17 00:00:00 2001 From: Ophir Back <45718146+Kle0s@users.noreply.github.com> Date: Mon, 23 Jan 2023 13:11:20 +0200 Subject: [PATCH] feat(page): Adding support for referrerPolicy in `page.goto` (#9561) Issue: #9394 **What kind of change does this PR introduce?** Feature - Added support of the `referrerPolicy` parameter (from CDP's `Page.navigate`) to Puppeteer's `page.goto`. **Did you add tests for your changes?** No, currently it has no meaning that isn't browser implementation based, which might get broken in the future. If there are suggestions to tests, please let me know so I'll add them. **If relevant, did you update the documentation?** Yes, the documentation of the `goto` method has been updated. **Summary** I wanted to contribute to this project, which I used for testing on our environment and saw issue #9394, so I decided to resolve it :) **Does this PR introduce a breaking change?** No. I added the `referrerPolicy` as an optional parameter, which will use the HTTP header `Referrer-Policy` if not provided (much like the `referer` parameter) so it will not interfere. Co-authored-by: Ophir Back --- docs/api/puppeteer.frame.goto.md | 9 +++++---- docs/api/puppeteer.page.goto.md | 11 ++++++----- packages/puppeteer-core/src/api/Page.ts | 5 ++++- packages/puppeteer-core/src/common/Frame.ts | 14 +++++++++++++- packages/puppeteer-core/src/common/Page.ts | 2 +- test/src/navigation.spec.ts | 14 ++++++++++++++ 6 files changed, 43 insertions(+), 12 deletions(-) diff --git a/docs/api/puppeteer.frame.goto.md b/docs/api/puppeteer.frame.goto.md index b1265055..ff377102 100644 --- a/docs/api/puppeteer.frame.goto.md +++ b/docs/api/puppeteer.frame.goto.md @@ -14,6 +14,7 @@ class Frame { url: string, options?: { referer?: string; + referrerPolicy?: string; timeout?: number; waitUntil?: PuppeteerLifeCycleEvent | PuppeteerLifeCycleEvent[]; } @@ -23,10 +24,10 @@ class Frame { ## Parameters -| Parameter | Type | Description | -| --------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| url | string | the URL to navigate the frame to. This should include the scheme, e.g. https://. | -| options | { referer?: string; timeout?: number; waitUntil?: [PuppeteerLifeCycleEvent](./puppeteer.puppeteerlifecycleevent.md) \| [PuppeteerLifeCycleEvent](./puppeteer.puppeteerlifecycleevent.md)\[\]; } | (Optional) navigation options. waitUntil is useful to define when the navigation should be considered successful - see the docs for [PuppeteerLifeCycleEvent](./puppeteer.puppeteerlifecycleevent.md) for more details. | +| Parameter | Type | Description | +| --------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| url | string | the URL to navigate the frame to. This should include the scheme, e.g. https://. | +| options | { referer?: string; referrerPolicy?: string; timeout?: number; waitUntil?: [PuppeteerLifeCycleEvent](./puppeteer.puppeteerlifecycleevent.md) \| [PuppeteerLifeCycleEvent](./puppeteer.puppeteerlifecycleevent.md)\[\]; } | (Optional) navigation options. waitUntil is useful to define when the navigation should be considered successful - see the docs for [PuppeteerLifeCycleEvent](./puppeteer.puppeteerlifecycleevent.md) for more details. | **Returns:** diff --git a/docs/api/puppeteer.page.goto.md b/docs/api/puppeteer.page.goto.md index 91860ea1..4b4ec1ce 100644 --- a/docs/api/puppeteer.page.goto.md +++ b/docs/api/puppeteer.page.goto.md @@ -12,6 +12,7 @@ class Page { url: string, options?: WaitForOptions & { referer?: string; + referrerPolicy?: string; } ): Promise; } @@ -19,10 +20,10 @@ class Page { ## Parameters -| Parameter | Type | Description | -| --------- | --------------------------------------------------------------------------- | ---------------------------------------------------------------------------------- | -| url | string | URL to navigate page to. The URL should include scheme, e.g. https:// | -| options | [WaitForOptions](./puppeteer.waitforoptions.md) & { referer?: string; } | (Optional) Navigation Parameter | +| Parameter | Type | Description | +| --------- | ---------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------- | +| url | string | URL to navigate page to. The URL should include scheme, e.g. https:// | +| options | [WaitForOptions](./puppeteer.waitforoptions.md) & { referer?: string; referrerPolicy?: string; } | (Optional) Navigation Parameter | **Returns:** @@ -38,7 +39,7 @@ The argument `options` might have the following properties: - `waitUntil`:When to consider navigation succeeded, defaults to `load`. Given an array of event strings, navigation is considered to be successful after all events have been fired. Events can be either:
- `load` : consider navigation to be finished when the load event is fired.
- `domcontentloaded` : consider navigation to be finished when the DOMContentLoaded event is fired.
- `networkidle0` : consider navigation to be finished when there are no more than 0 network connections for at least `500` ms.
- `networkidle2` : consider navigation to be finished when there are no more than 2 network connections for at least `500` ms. -- `referer` : Referer header value. If provided it will take preference over the referer header value set by [page.setExtraHTTPHeaders()](./puppeteer.page.setextrahttpheaders.md). +- `referer` : Referer header value. If provided it will take preference over the referer header value set by [page.setExtraHTTPHeaders()](./puppeteer.page.setextrahttpheaders.md).
- `referrerPolicy` : ReferrerPolicy. If provided it will take preference over the referer-policy header value set by [page.setExtraHTTPHeaders()](./puppeteer.page.setextrahttpheaders.md). `page.goto` will throw an error if: diff --git a/packages/puppeteer-core/src/api/Page.ts b/packages/puppeteer-core/src/api/Page.ts index 85aa3d88..31f6e9f4 100644 --- a/packages/puppeteer-core/src/api/Page.ts +++ b/packages/puppeteer-core/src/api/Page.ts @@ -1381,6 +1381,9 @@ export class Page extends EventEmitter { * * - `referer` : Referer header value. If provided it will take preference * over the referer header value set by + * {@link Page.setExtraHTTPHeaders |page.setExtraHTTPHeaders()}.
+ * - `referrerPolicy` : ReferrerPolicy. If provided it will take preference + * over the referer-policy header value set by * {@link Page.setExtraHTTPHeaders |page.setExtraHTTPHeaders()}. * * `page.goto` will throw an error if: @@ -1408,7 +1411,7 @@ export class Page extends EventEmitter { */ async goto( url: string, - options?: WaitForOptions & {referer?: string} + options?: WaitForOptions & {referer?: string; referrerPolicy?: string} ): Promise; async goto(): Promise { throw new Error('Not implemented'); diff --git a/packages/puppeteer-core/src/common/Frame.ts b/packages/puppeteer-core/src/common/Frame.ts index 63a28075..cbd1097a 100644 --- a/packages/puppeteer-core/src/common/Frame.ts +++ b/packages/puppeteer-core/src/common/Frame.ts @@ -289,12 +289,16 @@ export class Frame { url: string, options: { referer?: string; + referrerPolicy?: string; timeout?: number; waitUntil?: PuppeteerLifeCycleEvent | PuppeteerLifeCycleEvent[]; } = {} ): Promise { const { referer = this._frameManager.networkManager.extraHTTPHeaders()['referer'], + referrerPolicy = this._frameManager.networkManager.extraHTTPHeaders()[ + 'referer-policy' + ], waitUntil = ['load'], timeout = this._frameManager.timeoutSettings.navigationTimeout(), } = options; @@ -307,7 +311,13 @@ export class Frame { timeout ); let error = await Promise.race([ - navigate(this.#client, url, referer, this._id), + navigate( + this.#client, + url, + referer, + referrerPolicy as Protocol.Page.ReferrerPolicy, + this._id + ), watcher.timeoutOrTerminationPromise(), ]); if (!error) { @@ -332,6 +342,7 @@ export class Frame { client: CDPSession, url: string, referrer: string | undefined, + referrerPolicy: Protocol.Page.ReferrerPolicy | undefined, frameId: string ): Promise { try { @@ -339,6 +350,7 @@ export class Frame { url, referrer, frameId, + referrerPolicy, }); ensureNewDocumentNavigation = !!response.loaderId; return response.errorText diff --git a/packages/puppeteer-core/src/common/Page.ts b/packages/puppeteer-core/src/common/Page.ts index 388fc7f0..f86abf6f 100644 --- a/packages/puppeteer-core/src/common/Page.ts +++ b/packages/puppeteer-core/src/common/Page.ts @@ -893,7 +893,7 @@ export class CDPPage extends Page { override async goto( url: string, - options: WaitForOptions & {referer?: string} = {} + options: WaitForOptions & {referer?: string; referrerPolicy?: string} = {} ): Promise { return await this.#frameManager.mainFrame().goto(url, options); } diff --git a/test/src/navigation.spec.ts b/test/src/navigation.spec.ts index 73013c68..fea8015c 100644 --- a/test/src/navigation.spec.ts +++ b/test/src/navigation.spec.ts @@ -494,6 +494,20 @@ describe('navigation', function () { // Make sure subresources do not inherit referer. expect(request2.headers['referer']).toBe(server.PREFIX + '/grid.html'); }); + + it('should send referer policy', async () => { + const {page, server} = getTestState(); + + const [request1, request2] = await Promise.all([ + server.waitForRequest('/grid.html'), + server.waitForRequest('/digits/1.png'), + page.goto(server.PREFIX + '/grid.html', { + referrerPolicy: 'no-referer', + }), + ]); + expect(request1.headers['referer']).toBeUndefined(); + expect(request2.headers['referer']).toBe(server.PREFIX + '/grid.html'); + }); }); describe('Page.waitForNavigation', function () {