diff --git a/docs/api/index.md b/docs/api/index.md index 7b44f36e489..0e6ecdc426d 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -135,6 +135,7 @@ sidebar_label: API | [SnapshotOptions](./puppeteer.snapshotoptions.md) | | | [TracingOptions](./puppeteer.tracingoptions.md) | | | [Viewport](./puppeteer.viewport.md) | | +| [WaitForNetworkIdleOptions](./puppeteer.waitfornetworkidleoptions.md) | | | [WaitForOptions](./puppeteer.waitforoptions.md) | | | [WaitForSelectorOptions](./puppeteer.waitforselectoroptions.md) | | | [WaitForTargetOptions](./puppeteer.waitfortargetoptions.md) | | diff --git a/docs/api/puppeteer.page.md b/docs/api/puppeteer.page.md index d487cc1cb58..4061546b4cd 100644 --- a/docs/api/puppeteer.page.md +++ b/docs/api/puppeteer.page.md @@ -157,7 +157,7 @@ page.off('request', logRequest); | [waitForFrame](./puppeteer.page.waitforframe.md) | | Waits for a frame matching the given conditions to appear. | | [waitForFunction](./puppeteer.page.waitforfunction.md) | | Waits for the provided function, pageFunction, to return a truthy value when evaluated in the page's context. | | [waitForNavigation](./puppeteer.page.waitfornavigation.md) | | Waits for the page to navigate to a new URL or to reload. It is useful when you run code that will indirectly cause the page to navigate. | -| [waitForNetworkIdle](./puppeteer.page.waitfornetworkidle.md) | | | +| [waitForNetworkIdle](./puppeteer.page.waitfornetworkidle.md) | | Waits for the network to be idle. | | [waitForRequest](./puppeteer.page.waitforrequest.md) | | | | [waitForResponse](./puppeteer.page.waitforresponse.md) | | | | [waitForSelector](./puppeteer.page.waitforselector.md) | | Wait for the selector to appear in page. If at the moment of calling the method the selector already exists, the method will return immediately. If the selector doesn't appear after the timeout milliseconds of waiting, the function will throw. | diff --git a/docs/api/puppeteer.page.waitfornetworkidle.md b/docs/api/puppeteer.page.waitfornetworkidle.md index 7aba83f87f2..6d060482670 100644 --- a/docs/api/puppeteer.page.waitfornetworkidle.md +++ b/docs/api/puppeteer.page.waitfornetworkidle.md @@ -4,25 +4,24 @@ sidebar_label: Page.waitForNetworkIdle # Page.waitForNetworkIdle() method +Waits for the network to be idle. + #### Signature: ```typescript class Page { - abstract waitForNetworkIdle(options?: { - idleTime?: number; - timeout?: number; - }): Promise; + waitForNetworkIdle(options?: WaitForNetworkIdleOptions): Promise; } ``` ## Parameters -| Parameter | Type | Description | -| --------- | -------------------------------------------------- | ---------------------------------------- | -| options | { idleTime?: number; timeout?: number; } | _(Optional)_ Optional waiting parameters | +| Parameter | Type | Description | +| --------- | --------------------------------------------------------------------- | --------------------------------------------------- | +| options | [WaitForNetworkIdleOptions](./puppeteer.waitfornetworkidleoptions.md) | _(Optional)_ Options to configure waiting behavior. | **Returns:** Promise<void> -Promise which resolves when network is idle +A promise which resolves once the network is idle. diff --git a/docs/api/puppeteer.waitfornetworkidleoptions.md b/docs/api/puppeteer.waitfornetworkidleoptions.md new file mode 100644 index 00000000000..ffd53e89c34 --- /dev/null +++ b/docs/api/puppeteer.waitfornetworkidleoptions.md @@ -0,0 +1,20 @@ +--- +sidebar_label: WaitForNetworkIdleOptions +--- + +# WaitForNetworkIdleOptions interface + +#### Signature: + +```typescript +export interface WaitForNetworkIdleOptions extends WaitTimeoutOptions +``` + +**Extends:** [WaitTimeoutOptions](./puppeteer.waittimeoutoptions.md) + +## Properties + +| Property | Modifiers | Type | Description | Default | +| ----------- | --------------------- | ------ | --------------------------------------------------------------------------- | ---------------- | +| concurrency | optional | number | Maximum number concurrent of network connections to be considered inactive. | 0 | +| idleTime | optional | number | Time (in milliseconds) the network should be idle. | 500 | diff --git a/packages/puppeteer-core/src/api/Page.ts b/packages/puppeteer-core/src/api/Page.ts index f9e3b4c5983..deb04628fdc 100644 --- a/packages/puppeteer-core/src/api/Page.ts +++ b/packages/puppeteer-core/src/api/Page.ts @@ -9,7 +9,8 @@ import type {Readable} from 'stream'; import type {Protocol} from 'devtools-protocol'; import { - delay, + concat, + EMPTY, filter, filterAsync, first, @@ -17,23 +18,22 @@ import { from, map, merge, + mergeMap, of, + race, raceWith, startWith, switchMap, + takeUntil, + timer, type Observable, } from '../../third_party/rxjs/rxjs.js'; import type {HTTPRequest} from '../api/HTTPRequest.js'; import type {HTTPResponse} from '../api/HTTPResponse.js'; -import type {BidiNetworkManager} from '../bidi/NetworkManager.js'; import type {Accessibility} from '../cdp/Accessibility.js'; import type {Coverage} from '../cdp/Coverage.js'; import type {DeviceRequestPrompt} from '../cdp/DeviceRequestPrompt.js'; -import type { - NetworkManager as CdpNetworkManager, - Credentials, - NetworkConditions, -} from '../cdp/NetworkManager.js'; +import type {Credentials, NetworkConditions} from '../cdp/NetworkManager.js'; import type {Tracing} from '../cdp/Tracing.js'; import type {ConsoleMessage} from '../common/ConsoleMessage.js'; import type {Device} from '../common/Device.js'; @@ -45,7 +45,6 @@ import { type Handler, } from '../common/EventEmitter.js'; import type {FileChooser} from '../common/FileChooser.js'; -import {NetworkManagerEvent} from '../common/NetworkManagerEvents.js'; import type {PDFOptions} from '../common/PDFOptions.js'; import {TimeoutSettings} from '../common/TimeoutSettings.js'; import type { @@ -61,6 +60,7 @@ import { fromEmitterEvent, importFSPromises, isString, + NETWORK_IDLE_TIME, timeout, withSourcePuppeteerURLIfNone, } from '../common/util.js'; @@ -126,6 +126,24 @@ export interface Metrics { JSHeapTotalSize?: number; } +/** + * @public + */ +export interface WaitForNetworkIdleOptions extends WaitTimeoutOptions { + /** + * Time (in milliseconds) the network should be idle. + * + * @defaultValue `500` + */ + idleTime?: number; + /** + * Maximum number concurrent of network connections to be considered inactive. + * + * @defaultValue `0` + */ + concurrency?: number; +} + /** * @public */ @@ -586,11 +604,48 @@ export abstract class Page extends EventEmitter { #requestHandlers = new WeakMap, Handler>(); + #requestsInFlight = 0; + #inflight$: Observable; + /** * @internal */ constructor() { super(); + + this.#inflight$ = fromEmitterEvent(this, PageEvent.Request).pipe( + takeUntil(fromEmitterEvent(this, PageEvent.Close)), + mergeMap(request => { + return concat( + of(1), + race( + fromEmitterEvent(this, PageEvent.Response).pipe( + filter(response => { + return response.request()._requestId === request._requestId; + }) + ), + fromEmitterEvent(this, PageEvent.RequestFailed).pipe( + filter(failure => { + return failure._requestId === request._requestId; + }) + ), + fromEmitterEvent(this, PageEvent.RequestFinished).pipe( + filter(success => { + return success._requestId === request._requestId; + }) + ) + ).pipe( + map(() => { + return -1; + }) + ) + ); + }) + ); + + this.#inflight$.subscribe(count => { + this.#requestsInFlight += count; + }); } /** @@ -1699,34 +1754,45 @@ export abstract class Page extends EventEmitter { } /** - * @param options - Optional waiting parameters - * @returns Promise which resolves when network is idle + * Waits for the network to be idle. + * + * @param options - Options to configure waiting behavior. + * @returns A promise which resolves once the network is idle. */ - abstract waitForNetworkIdle(options?: { - idleTime?: number; - timeout?: number; - }): Promise; + waitForNetworkIdle(options: WaitForNetworkIdleOptions = {}): Promise { + return firstValueFrom(this.waitForNetworkIdle$(options)); + } /** * @internal */ - _waitForNetworkIdle( - networkManager: BidiNetworkManager | CdpNetworkManager, - idleTime: number, - requestsInFlight = 0 + waitForNetworkIdle$( + options: WaitForNetworkIdleOptions = {} ): Observable { - return merge( - fromEmitterEvent(networkManager, NetworkManagerEvent.Request), - fromEmitterEvent(networkManager, NetworkManagerEvent.Response), - fromEmitterEvent(networkManager, NetworkManagerEvent.RequestFailed) - ).pipe( - startWith(undefined), - filter(() => { - return networkManager.inFlightRequestsCount() <= requestsInFlight; - }), + const { + timeout: ms = this._timeoutSettings.timeout(), + idleTime = NETWORK_IDLE_TIME, + concurrency = 0, + } = options; + + return this.#inflight$.pipe( + startWith(this.#requestsInFlight), switchMap(() => { - return of(undefined).pipe(delay(idleTime)); - }) + if (this.#requestsInFlight > concurrency) { + return EMPTY; + } else { + return timer(idleTime); + } + }), + map(() => {}), + raceWith( + timeout(ms), + fromEmitterEvent(this, PageEvent.Close).pipe( + map(() => { + throw new TargetCloseError('Page closed!'); + }) + ) + ) ); } diff --git a/packages/puppeteer-core/src/bidi/Frame.ts b/packages/puppeteer-core/src/bidi/Frame.ts index 5672aaf0567..1638c2cbdff 100644 --- a/packages/puppeteer-core/src/bidi/Frame.ts +++ b/packages/puppeteer-core/src/bidi/Frame.ts @@ -14,6 +14,7 @@ import { map, merge, raceWith, + zip, } from '../../third_party/rxjs/rxjs.js'; import type {CDPSession} from '../api/CDPSession.js'; import type {ElementHandle} from '../api/ElementHandle.js'; @@ -27,7 +28,12 @@ import type {WaitForSelectorOptions} from '../api/Page.js'; import {UnsupportedOperation} from '../common/Errors.js'; import type {TimeoutSettings} from '../common/TimeoutSettings.js'; import type {Awaitable, NodeFor} from '../common/types.js'; -import {fromEmitterEvent, timeout, UTILITY_WORLD_NAME} from '../common/util.js'; +import { + fromEmitterEvent, + NETWORK_IDLE_TIME, + timeout, + UTILITY_WORLD_NAME, +} from '../common/util.js'; import {Deferred} from '../util/Deferred.js'; import {disposeSymbol} from '../util/disposable.js'; @@ -128,21 +134,33 @@ export class BidiFrame extends Frame { const [readiness, networkIdle] = getBiDiReadinessState(waitUntil); - const response = await firstValueFrom( - this.#page - ._waitWithNetworkIdle( - this.#context.connection.send('browsingContext.navigate', { - context: this.#context.id, - url, - wait: readiness, - }), - networkIdle - ) - .pipe(raceWith(timeout(ms), from(this.#abortDeferred.valueOrThrow()))) - .pipe(rewriteNavigationError(url, ms)) + const result$ = zip( + from( + this.#context.connection.send('browsingContext.navigate', { + context: this.#context.id, + url, + wait: readiness, + }) + ), + ...(networkIdle !== null + ? [ + this.#page.waitForNetworkIdle$({ + timeout: ms, + concurrency: networkIdle === 'networkidle2' ? 2 : 0, + idleTime: NETWORK_IDLE_TIME, + }), + ] + : []) + ).pipe( + map(([{result}]) => { + return result; + }), + raceWith(timeout(ms), from(this.#abortDeferred.valueOrThrow())), + rewriteNavigationError(url, ms) ); - return this.#page.getNavigationResponse(response?.result.navigation); + const result = await firstValueFrom(result$); + return this.#page.getNavigationResponse(result.navigation); } @throwIfDetached @@ -157,22 +175,30 @@ export class BidiFrame extends Frame { const [waitEvent, networkIdle] = getBiDiLifecycleEvent(waitUntil); - await firstValueFrom( - this.#page - ._waitWithNetworkIdle( - forkJoin([ - fromEmitterEvent(this.#context, waitEvent).pipe(first()), - from(this.setFrameContent(html)), - ]).pipe( - map(() => { - return null; - }) - ), - networkIdle - ) - .pipe(raceWith(timeout(ms), from(this.#abortDeferred.valueOrThrow()))) - .pipe(rewriteNavigationError('setContent', ms)) + const result$ = zip( + forkJoin([ + fromEmitterEvent(this.#context, waitEvent).pipe(first()), + from(this.setFrameContent(html)), + ]).pipe( + map(() => { + return null; + }) + ), + ...(networkIdle !== null + ? [ + this.#page.waitForNetworkIdle$({ + timeout: ms, + concurrency: networkIdle === 'networkidle2' ? 2 : 0, + idleTime: NETWORK_IDLE_TIME, + }), + ] + : []) + ).pipe( + raceWith(timeout(ms), from(this.#abortDeferred.valueOrThrow())), + rewriteNavigationError('setContent', ms) ); + + await firstValueFrom(result$); } context(): BrowsingContext { @@ -190,7 +216,7 @@ export class BidiFrame extends Frame { const [waitUntilEvent, networkIdle] = getBiDiLifecycleEvent(waitUntil); - const navigatedObservable = merge( + const navigation$ = merge( forkJoin([ fromEmitterEvent( this.#context, @@ -211,13 +237,26 @@ export class BidiFrame extends Frame { }) ); - const response = await firstValueFrom( - this.#page - ._waitWithNetworkIdle(navigatedObservable, networkIdle) - .pipe(raceWith(timeout(ms), from(this.#abortDeferred.valueOrThrow()))) + const result$ = zip( + navigation$, + ...(networkIdle !== null + ? [ + this.#page.waitForNetworkIdle$({ + timeout: ms, + concurrency: networkIdle === 'networkidle2' ? 2 : 0, + idleTime: NETWORK_IDLE_TIME, + }), + ] + : []) + ).pipe( + map(([{result}]) => { + return result; + }), + raceWith(timeout(ms), from(this.#abortDeferred.valueOrThrow())) ); - return this.#page.getNavigationResponse(response?.result.navigation); + const result = await firstValueFrom(result$); + return this.#page.getNavigationResponse(result.navigation); } override waitForDevicePrompt(): never { diff --git a/packages/puppeteer-core/src/bidi/Page.ts b/packages/puppeteer-core/src/bidi/Page.ts index 567e2944f4c..053d23b63a1 100644 --- a/packages/puppeteer-core/src/bidi/Page.ts +++ b/packages/puppeteer-core/src/bidi/Page.ts @@ -9,14 +9,12 @@ import type {Readable} from 'stream'; import type * as Bidi from 'chromium-bidi/lib/cjs/protocol/protocol.js'; import type Protocol from 'devtools-protocol'; -import type {Observable, ObservableInput} from '../../third_party/rxjs/rxjs.js'; import { - first, firstValueFrom, - forkJoin, from, map, raceWith, + zip, } from '../../third_party/rxjs/rxjs.js'; import type {CDPSession} from '../api/CDPSession.js'; import type {BoundingBox} from '../api/ElementHandle.js'; @@ -75,7 +73,6 @@ import type {BidiHTTPRequest} from './HTTPRequest.js'; import type {BidiHTTPResponse} from './HTTPResponse.js'; import {BidiKeyboard, BidiMouse, BidiTouchscreen} from './Input.js'; import type {BidiJSHandle} from './JSHandle.js'; -import type {BiDiNetworkIdle} from './lifecycle.js'; import {getBiDiReadinessState, rewriteNavigationError} from './lifecycle.js'; import {BidiNetworkManager} from './NetworkManager.js'; import {createBidiHandle} from './Realm.js'; @@ -497,19 +494,32 @@ export class BidiPage extends Page { const [readiness, networkIdle] = getBiDiReadinessState(waitUntil); - const response = await firstValueFrom( - this._waitWithNetworkIdle( + const result$ = zip( + from( this.#connection.send('browsingContext.reload', { context: this.mainFrame()._id, wait: readiness, - }), - networkIdle - ) - .pipe(raceWith(timeout(ms), from(this.#closedDeferred.valueOrThrow()))) - .pipe(rewriteNavigationError(this.url(), ms)) + }) + ), + ...(networkIdle !== null + ? [ + this.waitForNetworkIdle$({ + timeout: ms, + concurrency: networkIdle === 'networkidle2' ? 2 : 0, + idleTime: NETWORK_IDLE_TIME, + }), + ] + : []) + ).pipe( + map(([{result}]) => { + return result; + }), + raceWith(timeout(ms), from(this.#closedDeferred.valueOrThrow())), + rewriteNavigationError(this.url(), ms) ); - return this.getNavigationResponse(response?.result.navigation); + const result = await firstValueFrom(result$); + return this.getNavigationResponse(result.navigation); } override setDefaultNavigationTimeout(timeout: number): void { @@ -701,48 +711,6 @@ export class BidiPage extends Page { return data; } - override async waitForNetworkIdle( - options: {idleTime?: number; timeout?: number} = {} - ): Promise { - const { - idleTime = NETWORK_IDLE_TIME, - timeout: ms = this._timeoutSettings.timeout(), - } = options; - - await firstValueFrom( - this._waitForNetworkIdle(this.#networkManager, idleTime).pipe( - raceWith(timeout(ms), from(this.#closedDeferred.valueOrThrow())) - ) - ); - } - - /** @internal */ - _waitWithNetworkIdle( - observableInput: ObservableInput<{ - result: Bidi.BrowsingContext.NavigateResult; - } | null>, - networkIdle: BiDiNetworkIdle - ): Observable<{ - result: Bidi.BrowsingContext.NavigateResult; - } | null> { - const delay = networkIdle - ? this._waitForNetworkIdle( - this.#networkManager, - NETWORK_IDLE_TIME, - networkIdle === 'networkidle0' ? 0 : 2 - ) - : from(Promise.resolve()); - - return forkJoin([ - from(observableInput).pipe(first()), - delay.pipe(first()), - ]).pipe( - map(([response]) => { - return response; - }) - ); - } - override async createCDPSession(): Promise { const {sessionId} = await this.mainFrame() .context() diff --git a/packages/puppeteer-core/src/cdp/Page.ts b/packages/puppeteer-core/src/cdp/Page.ts index db6b456bf33..491637f0ea3 100644 --- a/packages/puppeteer-core/src/cdp/Page.ts +++ b/packages/puppeteer-core/src/cdp/Page.ts @@ -42,7 +42,6 @@ import { evaluationString, getReadableAsBuffer, getReadableFromProtocolStream, - NETWORK_IDLE_TIME, parsePDFOptions, timeout, validateDialogType, @@ -909,24 +908,6 @@ export class CdpPage extends Page { return await this.target().createCDPSession(); } - override async waitForNetworkIdle( - options: {idleTime?: number; timeout?: number} = {} - ): Promise { - const { - idleTime = NETWORK_IDLE_TIME, - timeout: ms = this._timeoutSettings.timeout(), - } = options; - - await firstValueFrom( - this._waitForNetworkIdle( - this.#frameManager.networkManager, - idleTime - ).pipe( - raceWith(timeout(ms), from(this.#sessionCloseDeferred.valueOrThrow())) - ) - ); - } - override async goBack( options: WaitForOptions = {} ): Promise { diff --git a/packages/puppeteer-core/third_party/rxjs/rxjs.ts b/packages/puppeteer-core/third_party/rxjs/rxjs.ts index 9bb05a29848..b8b64788ae5 100644 --- a/packages/puppeteer-core/third_party/rxjs/rxjs.ts +++ b/packages/puppeteer-core/third_party/rxjs/rxjs.ts @@ -6,6 +6,7 @@ export { bufferCount, catchError, + concat, concatMap, defaultIfEmpty, defer, @@ -37,6 +38,7 @@ export { tap, throwIfEmpty, timer, + zip, } from 'rxjs'; export type * from 'rxjs'; diff --git a/test/src/page.spec.ts b/test/src/page.spec.ts index e6f64fd1c05..79fc69ebbc3 100644 --- a/test/src/page.spec.ts +++ b/test/src/page.spec.ts @@ -851,21 +851,16 @@ describe('Page', function () { return Date.now(); }), page - .evaluate(() => { - return (async () => { - await Promise.all([ - fetch('/digits/1.png'), - fetch('/digits/2.png'), - ]); - await new Promise(resolve => { - return setTimeout(resolve, 200); - }); - await fetch('/digits/3.png'); - await new Promise(resolve => { - return setTimeout(resolve, 200); - }); - await fetch('/digits/4.png'); - })(); + .evaluate(async () => { + await Promise.all([fetch('/digits/1.png'), fetch('/digits/2.png')]); + await new Promise(resolve => { + return setTimeout(resolve, 200); + }); + await fetch('/digits/3.png'); + await new Promise(resolve => { + return setTimeout(resolve, 200); + }); + await fetch('/digits/4.png'); }) .then(() => { return Date.now(); @@ -938,6 +933,34 @@ describe('Page', function () { expect(error).toBe(false); }); + it('should work with delayed response', async () => { + const {page, server} = await getTestState(); + await page.goto(server.EMPTY_PAGE); + let response!: ServerResponse; + server.setRoute('/fetch-request-b.js', (_req, res) => { + response = res; + }); + const t0 = Date.now(); + const [t1, t2] = await Promise.all([ + page.waitForNetworkIdle({idleTime: 100}).then(() => { + return Date.now(); + }), + new Promise(res => { + setTimeout(() => { + response.end(); + res(Date.now()); + }, 300); + }), + page.evaluate(async () => { + await fetch('/fetch-request-b.js'); + }), + ]); + expect(t1).toBeGreaterThan(t2); + // request finished + idle time. + expect(t1 - t0).toBeGreaterThan(400); + // request finished + idle time - request finished. + expect(t1 - t2).toBeGreaterThanOrEqual(100); + }); }); describe('Page.exposeFunction', function () {