refactor: use RxJS for waitForHttp (#11133)

This commit is contained in:
Nikolay Vitkov 2023-10-13 12:35:43 +02:00 committed by GitHub
parent 2c7d802d87
commit 8290dc9de9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 156 additions and 178 deletions

View File

@ -40,11 +40,10 @@ import type {BidiNetworkManager} from '../bidi/NetworkManager.js';
import type {Accessibility} from '../cdp/Accessibility.js';
import type {Coverage} from '../cdp/Coverage.js';
import type {DeviceRequestPrompt} from '../cdp/DeviceRequestPrompt.js';
import {
NetworkManagerEvent,
type NetworkManager as CdpNetworkManager,
type Credentials,
type NetworkConditions,
import type {
NetworkManager as CdpNetworkManager,
Credentials,
NetworkConditions,
} from '../cdp/NetworkManager.js';
import type {Tracing} from '../cdp/Tracing.js';
import type {WebWorker} from '../cdp/WebWorker.js';
@ -58,12 +57,14 @@ import {
type Handler,
} from '../common/EventEmitter.js';
import type {FileChooser} from '../common/FileChooser.js';
import {NetworkManagerEvent} from '../common/NetworkManagerEvents.js';
import {
paperFormats,
type LowerCasePaperFormat,
type ParsedPDFOptions,
type PDFOptions,
} from '../common/PDFOptions.js';
import {TimeoutSettings} from '../common/TimeoutSettings.js';
import type {
Awaitable,
EvaluateFunc,
@ -603,6 +604,10 @@ export abstract class Page extends EventEmitter<PageEvents> {
* @internal
*/
_isDragging = false;
/**
* @internal
*/
_timeoutSettings = new TimeoutSettings();
#requestHandlers = new WeakMap<Handler<HTTPRequest>, Handler<HTTPRequest>>();

View File

@ -16,12 +16,11 @@
import type * as Bidi from 'chromium-bidi/lib/cjs/protocol/protocol.js';
import {NetworkManagerEvent} from '../cdp/NetworkManager.js';
import {EventEmitter, EventSubscription} from '../common/EventEmitter.js';
import {
EventEmitter,
EventSubscription,
type EventType,
} from '../common/EventEmitter.js';
NetworkManagerEvent,
type NetworkManagerEvents,
} from '../common/NetworkManagerEvents.js';
import {DisposableStack} from '../util/disposable.js';
import type {BidiConnection} from './Connection.js';
@ -33,18 +32,7 @@ import type {BidiPage} from './Page.js';
/**
* @internal
*/
export interface BidiNetworkManagerEvents extends Record<EventType, unknown> {
[NetworkManagerEvent.Request]: BidiHTTPRequest;
[NetworkManagerEvent.RequestServedFromCache]: BidiHTTPRequest;
[NetworkManagerEvent.Response]: BidiHTTPResponse;
[NetworkManagerEvent.RequestFailed]: BidiHTTPRequest;
[NetworkManagerEvent.RequestFinished]: BidiHTTPRequest;
}
/**
* @internal
*/
export class BidiNetworkManager extends EventEmitter<BidiNetworkManagerEvents> {
export class BidiNetworkManager extends EventEmitter<NetworkManagerEvents> {
#connection: BidiConnection;
#page: BidiPage;
#subscriptions = new DisposableStack();

View File

@ -33,7 +33,6 @@ import {Accessibility} from '../cdp/Accessibility.js';
import {Coverage} from '../cdp/Coverage.js';
import {EmulationManager as CdpEmulationManager} from '../cdp/EmulationManager.js';
import {FrameTree} from '../cdp/FrameTree.js';
import {NetworkManagerEvent} from '../cdp/NetworkManager.js';
import {Tracing} from '../cdp/Tracing.js';
import {
ConsoleMessage,
@ -45,15 +44,14 @@ import {
TimeoutError,
} from '../common/Errors.js';
import type {Handler} from '../common/EventEmitter.js';
import {NetworkManagerEvent} from '../common/NetworkManagerEvents.js';
import type {PDFOptions} from '../common/PDFOptions.js';
import {TimeoutSettings} from '../common/TimeoutSettings.js';
import type {Awaitable} from '../common/types.js';
import {
debugError,
evaluationString,
isString,
validateDialogType,
waitForEvent,
waitForHTTP,
waitWithTimeout,
} from '../common/util.js';
import type {Viewport} from '../common/Viewport.js';
@ -87,7 +85,6 @@ import {BidiSerializer} from './Serializer.js';
*/
export class BidiPage extends Page {
#accessibility: Accessibility;
#timeoutSettings = new TimeoutSettings();
#connection: BidiConnection;
#frameTree = new FrameTree<BidiFrame>();
#networkManager: BidiNetworkManager;
@ -184,7 +181,7 @@ export class BidiPage extends Page {
const frame = new BidiFrame(
this,
this.#browsingContext,
this.#timeoutSettings,
this._timeoutSettings,
this.#browsingContext.parent
);
this.#frameTree.addFrame(frame);
@ -356,7 +353,7 @@ export class BidiPage extends Page {
const frame = new BidiFrame(
this,
context,
this.#timeoutSettings,
this._timeoutSettings,
context.parent
);
this.#frameTree.addFrame(frame);
@ -490,7 +487,7 @@ export class BidiPage extends Page {
): Promise<BidiHTTPResponse | null> {
const {
waitUntil = 'load',
timeout = this.#timeoutSettings.navigationTimeout(),
timeout = this._timeoutSettings.navigationTimeout(),
} = options;
const readinessState = lifeCycleToReadinessState.get(
@ -519,15 +516,15 @@ export class BidiPage extends Page {
}
override setDefaultNavigationTimeout(timeout: number): void {
this.#timeoutSettings.setDefaultNavigationTimeout(timeout);
this._timeoutSettings.setDefaultNavigationTimeout(timeout);
}
override setDefaultTimeout(timeout: number): void {
this.#timeoutSettings.setDefaultTimeout(timeout);
this._timeoutSettings.setDefaultTimeout(timeout);
}
override getDefaultTimeout(): number {
return this.#timeoutSettings.timeout();
return this._timeoutSettings.timeout();
}
override isJavaScriptEnabled(): boolean {
@ -691,21 +688,13 @@ export class BidiPage extends Page {
| ((req: BidiHTTPRequest) => boolean | Promise<boolean>),
options: {timeout?: number} = {}
): Promise<BidiHTTPRequest> {
const {timeout = this.#timeoutSettings.timeout()} = options;
return await waitForEvent(
const {timeout = this._timeoutSettings.timeout()} = options;
return await waitForHTTP(
this.#networkManager,
NetworkManagerEvent.Request,
async request => {
if (isString(urlOrPredicate)) {
return urlOrPredicate === request.url();
}
if (typeof urlOrPredicate === 'function') {
return !!(await urlOrPredicate(request));
}
return false;
},
urlOrPredicate,
timeout,
this.#closedDeferred.valueOrThrow()
this.#closedDeferred
);
}
@ -715,28 +704,20 @@ export class BidiPage extends Page {
| ((res: BidiHTTPResponse) => boolean | Promise<boolean>),
options: {timeout?: number} = {}
): Promise<BidiHTTPResponse> {
const {timeout = this.#timeoutSettings.timeout()} = options;
return await waitForEvent(
const {timeout = this._timeoutSettings.timeout()} = options;
return await waitForHTTP(
this.#networkManager,
NetworkManagerEvent.Response,
async response => {
if (isString(urlOrPredicate)) {
return urlOrPredicate === response.url();
}
if (typeof urlOrPredicate === 'function') {
return !!(await urlOrPredicate(response));
}
return false;
},
urlOrPredicate,
timeout,
this.#closedDeferred.valueOrThrow()
this.#closedDeferred
);
}
override async waitForNetworkIdle(
options: {idleTime?: number; timeout?: number} = {}
): Promise<void> {
const {idleTime = 500, timeout = this.#timeoutSettings.timeout()} = options;
const {idleTime = 500, timeout = this._timeoutSettings.timeout()} = options;
await this._waitForNetworkIdle(
this.#networkManager,

View File

@ -28,16 +28,18 @@ import {
type ResponseForRequest,
STATUS_TEXTS,
} from '../api/HTTPRequest.js';
import type {HTTPResponse} from '../api/HTTPResponse.js';
import type {ProtocolError} from '../common/Errors.js';
import {debugError, isString} from '../common/util.js';
import {assert} from '../util/assert.js';
import type {CdpHTTPResponse} from './HTTPResponse.js';
/**
* @internal
*/
export class CdpHTTPRequest extends HTTPRequest {
declare _redirectChain: CdpHTTPRequest[];
declare _response: CdpHTTPResponse | null;
#client: CDPSession;
#isNavigationRequest: boolean;
@ -191,7 +193,7 @@ export class CdpHTTPRequest extends HTTPRequest {
return this.#headers;
}
override response(): HTTPResponse | null {
override response(): CdpHTTPResponse | null {
return this._response;
}

View File

@ -17,17 +17,18 @@
import type Protocol from 'devtools-protocol';
import {type Frame, FrameEvent} from '../api/Frame.js';
import type {HTTPRequest} from '../api/HTTPRequest.js';
import type {HTTPResponse} from '../api/HTTPResponse.js';
import type {TimeoutError} from '../common/Errors.js';
import {EventSubscription} from '../common/EventEmitter.js';
import {NetworkManagerEvent} from '../common/NetworkManagerEvents.js';
import {assert} from '../util/assert.js';
import {Deferred} from '../util/Deferred.js';
import {DisposableStack} from '../util/disposable.js';
import type {CdpFrame} from './Frame.js';
import {FrameManagerEvent} from './FrameManager.js';
import type {CdpHTTPRequest} from './HTTPRequest.js';
import {type NetworkManager, NetworkManagerEvent} from './NetworkManager.js';
import type {NetworkManager} from './NetworkManager.js';
/**
* @public
@ -78,7 +79,7 @@ export class LifecycleWatcher {
#expectedLifecycle: ProtocolLifeCycleEvent[];
#frame: CdpFrame;
#timeout: number;
#navigationRequest: CdpHTTPRequest | null = null;
#navigationRequest: HTTPRequest | null = null;
#subscriptions = new DisposableStack();
#initialLoaderId: string;
@ -184,7 +185,7 @@ export class LifecycleWatcher {
this.#checkLifecycleComplete();
}
#onRequest(request: CdpHTTPRequest): void {
#onRequest(request: HTTPRequest): void {
if (request.frame() !== this.#frame || !request.isNavigationRequest()) {
return;
}
@ -199,7 +200,7 @@ export class LifecycleWatcher {
}
}
#onRequestFailed(request: CdpHTTPRequest): void {
#onRequestFailed(request: HTTPRequest): void {
if (this.#navigationRequest?._requestId !== request._requestId) {
return;
}

View File

@ -22,9 +22,10 @@ import type {CDPSessionEvents} from '../api/CDPSession.js';
import type {HTTPRequest} from '../api/HTTPRequest.js';
import type {HTTPResponse} from '../api/HTTPResponse.js';
import {EventEmitter} from '../common/EventEmitter.js';
import {NetworkManagerEvent} from '../common/NetworkManagerEvents.js';
import type {CdpFrame} from './Frame.js';
import {NetworkManager, NetworkManagerEvent} from './NetworkManager.js';
import {NetworkManager} from './NetworkManager.js';
// TODO: develop a helper to generate fake network events for attributes that
// are not relevant for the network manager to make tests shorter.

View File

@ -18,11 +18,11 @@ import type {Protocol} from 'devtools-protocol';
import {CDPSessionEvent, type CDPSession} from '../api/CDPSession.js';
import type {Frame} from '../api/Frame.js';
import {EventEmitter, EventSubscription} from '../common/EventEmitter.js';
import {
EventEmitter,
EventSubscription,
type EventType,
} from '../common/EventEmitter.js';
NetworkManagerEvent,
type NetworkManagerEvents,
} from '../common/NetworkManagerEvents.js';
import {debugError, isString} from '../common/util.js';
import {assert} from '../util/assert.js';
import {DisposableStack} from '../util/disposable.js';
@ -61,34 +61,6 @@ export interface InternalNetworkConditions extends NetworkConditions {
offline: boolean;
}
/**
* We use symbols to prevent any external parties listening to these events.
* They are internal to Puppeteer.
*
* @internal
*/
// eslint-disable-next-line @typescript-eslint/no-namespace
export namespace NetworkManagerEvent {
export const Request = Symbol('NetworkManager.Request');
export const RequestServedFromCache = Symbol(
'NetworkManager.RequestServedFromCache'
);
export const Response = Symbol('NetworkManager.Response');
export const RequestFailed = Symbol('NetworkManager.RequestFailed');
export const RequestFinished = Symbol('NetworkManager.RequestFinished');
}
/**
* @internal
*/
export interface CdpNetworkManagerEvents extends Record<EventType, unknown> {
[NetworkManagerEvent.Request]: CdpHTTPRequest;
[NetworkManagerEvent.RequestServedFromCache]: CdpHTTPRequest | undefined;
[NetworkManagerEvent.Response]: CdpHTTPResponse;
[NetworkManagerEvent.RequestFailed]: CdpHTTPRequest;
[NetworkManagerEvent.RequestFinished]: CdpHTTPRequest;
}
/**
* @internal
*/
@ -99,7 +71,7 @@ export interface FrameProvider {
/**
* @internal
*/
export class NetworkManager extends EventEmitter<CdpNetworkManagerEvents> {
export class NetworkManager extends EventEmitter<NetworkManagerEvents> {
#ignoreHTTPSErrors: boolean;
#frameManager: FrameProvider;
#networkEventManager = new NetworkEventManager();

View File

@ -43,8 +43,8 @@ import {
} from '../common/ConsoleMessage.js';
import {TargetCloseError} from '../common/Errors.js';
import {FileChooser} from '../common/FileChooser.js';
import {NetworkManagerEvent} from '../common/NetworkManagerEvents.js';
import type {PDFOptions} from '../common/PDFOptions.js';
import {TimeoutSettings} from '../common/TimeoutSettings.js';
import type {BindingPayload, HandleFor} from '../common/types.js';
import {
createClientError,
@ -52,11 +52,10 @@ import {
evaluationString,
getReadableAsBuffer,
getReadableFromProtocolStream,
isString,
pageBindingInitString,
validateDialogType,
valueFromRemoteObject,
waitForEvent,
waitForHTTP,
waitWithTimeout,
} from '../common/util.js';
import type {Viewport} from '../common/Viewport.js';
@ -79,11 +78,7 @@ import type {CdpFrame} from './Frame.js';
import {FrameManager, FrameManagerEvent} from './FrameManager.js';
import {CdpKeyboard, CdpMouse, CdpTouchscreen} from './Input.js';
import {MAIN_WORLD} from './IsolatedWorlds.js';
import {
NetworkManagerEvent,
type Credentials,
type NetworkConditions,
} from './NetworkManager.js';
import type {Credentials, NetworkConditions} from './NetworkManager.js';
import type {CdpTarget} from './Target.js';
import {TargetManagerEvent} from './TargetManager.js';
import {Tracing} from './Tracing.js';
@ -121,7 +116,6 @@ export class CdpPage extends Page {
#target: CdpTarget;
#keyboard: CdpKeyboard;
#mouse: CdpMouse;
#timeoutSettings = new TimeoutSettings();
#touchscreen: CdpTouchscreen;
#accessibility: Accessibility;
#frameManager: FrameManager;
@ -239,7 +233,7 @@ export class CdpPage extends Page {
client,
this,
ignoreHTTPSErrors,
this.#timeoutSettings
this._timeoutSettings
);
this.#emulationManager = new EmulationManager(client);
this.#tracing = new Tracing(client);
@ -402,7 +396,7 @@ export class CdpPage extends Page {
options: WaitTimeoutOptions = {}
): Promise<FileChooser> {
const needsEnable = this.#fileChooserDeferreds.size === 0;
const {timeout = this.#timeoutSettings.timeout()} = options;
const {timeout = this._timeoutSettings.timeout()} = options;
const deferred = Deferred.create<FileChooser>({
message: `Waiting for \`FileChooser\` failed: ${timeout}ms exceeded`,
timeout,
@ -522,15 +516,15 @@ export class CdpPage extends Page {
}
override setDefaultNavigationTimeout(timeout: number): void {
this.#timeoutSettings.setDefaultNavigationTimeout(timeout);
this._timeoutSettings.setDefaultNavigationTimeout(timeout);
}
override setDefaultTimeout(timeout: number): void {
this.#timeoutSettings.setDefaultTimeout(timeout);
this._timeoutSettings.setDefaultTimeout(timeout);
}
override getDefaultTimeout(): number {
return this.#timeoutSettings.timeout();
return this._timeoutSettings.timeout();
}
override async queryObjects<Prototype>(
@ -876,21 +870,13 @@ export class CdpPage extends Page {
urlOrPredicate: string | ((req: HTTPRequest) => boolean | Promise<boolean>),
options: {timeout?: number} = {}
): Promise<HTTPRequest> {
const {timeout = this.#timeoutSettings.timeout()} = options;
return await waitForEvent(
const {timeout = this._timeoutSettings.timeout()} = options;
return await waitForHTTP(
this.#frameManager.networkManager,
NetworkManagerEvent.Request,
async request => {
if (isString(urlOrPredicate)) {
return urlOrPredicate === request.url();
}
if (typeof urlOrPredicate === 'function') {
return !!(await urlOrPredicate(request));
}
return false;
},
urlOrPredicate,
timeout,
this.#sessionCloseDeferred.valueOrThrow()
this.#sessionCloseDeferred
);
}
@ -900,28 +886,20 @@ export class CdpPage extends Page {
| ((res: HTTPResponse) => boolean | Promise<boolean>),
options: {timeout?: number} = {}
): Promise<HTTPResponse> {
const {timeout = this.#timeoutSettings.timeout()} = options;
return await waitForEvent(
const {timeout = this._timeoutSettings.timeout()} = options;
return await waitForHTTP(
this.#frameManager.networkManager,
NetworkManagerEvent.Response,
async response => {
if (isString(urlOrPredicate)) {
return urlOrPredicate === response.url();
}
if (typeof urlOrPredicate === 'function') {
return !!(await urlOrPredicate(response));
}
return false;
},
urlOrPredicate,
timeout,
this.#sessionCloseDeferred.valueOrThrow()
this.#sessionCloseDeferred
);
}
override async waitForNetworkIdle(
options: {idleTime?: number; timeout?: number} = {}
): Promise<void> {
const {idleTime = 500, timeout = this.#timeoutSettings.timeout()} = options;
const {idleTime = 500, timeout = this._timeoutSettings.timeout()} = options;
await this._waitForNetworkIdle(
this.#frameManager.networkManager,

View File

@ -0,0 +1,48 @@
/**
* Copyright 2022 Google Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import type {HTTPRequest} from '../api/HTTPRequest.js';
import type {HTTPResponse} from '../api/HTTPResponse.js';
import type {EventType} from './EventEmitter.js';
/**
* We use symbols to prevent any external parties listening to these events.
* They are internal to Puppeteer.
*
* @internal
*/
// eslint-disable-next-line @typescript-eslint/no-namespace
export namespace NetworkManagerEvent {
export const Request = Symbol('NetworkManager.Request');
export const RequestServedFromCache = Symbol(
'NetworkManager.RequestServedFromCache'
);
export const Response = Symbol('NetworkManager.Response');
export const RequestFailed = Symbol('NetworkManager.RequestFailed');
export const RequestFinished = Symbol('NetworkManager.RequestFinished');
}
/**
* @internal
*/
export interface NetworkManagerEvents extends Record<EventType, unknown> {
[NetworkManagerEvent.Request]: HTTPRequest;
[NetworkManagerEvent.RequestServedFromCache]: HTTPRequest | undefined;
[NetworkManagerEvent.Response]: HTTPResponse;
[NetworkManagerEvent.RequestFailed]: HTTPRequest;
[NetworkManagerEvent.RequestFinished]: HTTPRequest;
}

View File

@ -15,6 +15,7 @@
*/
export * from './BrowserWebSocketTransport.js';
export * from './common.js';
export * from './Configuration.js';
export * from './ConnectionTransport.js';
export * from './ConsoleMessage.js';
@ -23,13 +24,15 @@ export * from './Debug.js';
export * from './Device.js';
export * from './Errors.js';
export * from './EventEmitter.js';
export * from './fetch.js';
export * from './FileChooser.js';
export * from './GetQueryHandler.js';
export * from './HandleIterator.js';
export * from './LazyArg.js';
export * from './NetworkManagerEvents.js';
export * from './PDFOptions.js';
export * from './PQueryHandler.js';
export * from './PierceQueryHandler.js';
export * from './PQueryHandler.js';
export * from './Product.js';
export * from './QueryHandler.js';
export * from './ScriptInjector.js';
@ -37,11 +40,9 @@ export * from './SecurityDetails.js';
export * from './TaskQueue.js';
export * from './TextQueryHandler.js';
export * from './TimeoutSettings.js';
export * from './types.js';
export * from './USKeyboardLayout.js';
export * from './util.js';
export * from './Viewport.js';
export * from './WaitTask.js';
export * from './XPathQueryHandler.js';
export * from './common.js';
export * from './fetch.js';
export * from './types.js';
export * from './util.js';

View File

@ -24,6 +24,11 @@ import {
NEVER,
timer,
type Observable,
firstValueFrom,
fromEvent,
filterAsync,
from,
raceWith,
} from '../../third_party/rxjs/rxjs.js';
import type {CDPSession} from '../api/CDPSession.js';
import type {Page} from '../api/Page.js';
@ -34,8 +39,8 @@ import {isErrorLike} from '../util/ErrorLike.js';
import {debug} from './Debug.js';
import {TimeoutError} from './Errors.js';
import {EventSubscription} from './EventEmitter.js';
import type {Awaitable} from './types.js';
import type {EventEmitter, EventType} from './EventEmitter.js';
import type {NetworkManagerEvents} from './NetworkManagerEvents.js';
/**
* @internal
@ -326,39 +331,6 @@ export const isDate = (obj: unknown): obj is Date => {
return typeof obj === 'object' && obj?.constructor === Date;
};
/**
* @internal
*/
export async function waitForEvent<T>(
// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
emitter: any,
eventName: string | symbol,
predicate: (event: T) => Awaitable<boolean>,
timeout: number,
abortPromise: Promise<Error> | Deferred<Error>
): Promise<T> {
const deferred = Deferred.create<T>({
message: `Timeout exceeded while waiting for event ${String(eventName)}`,
timeout,
});
// eslint-disable-next-line @typescript-eslint/no-unused-vars
using _ = new EventSubscription(emitter, eventName, async (event: any) => {
if (await predicate(event)) {
deferred.resolve(event);
}
});
try {
const response = await Deferred.race<T | Error>([deferred, abortPromise]);
if (isErrorLike(response)) {
throw response;
}
return response;
} catch (error) {
throw error;
}
}
/**
* @internal
*/
@ -632,3 +604,32 @@ export const SOURCE_URL_REGEX = /^[\040\t]*\/\/[@#] sourceURL=\s*(\S*?)\s*$/m;
export function getSourceUrlComment(url: string): string {
return `//# sourceURL=${url}`;
}
/**
* @internal
*/
export async function waitForHTTP<T extends {url(): string}>(
networkManager: EventEmitter<NetworkManagerEvents>,
eventName: EventType,
urlOrPredicate: string | ((res: T) => boolean | Promise<boolean>),
/** Time after the function will timeout */
ms: number,
cancelation: Deferred<never>
): Promise<T> {
return await firstValueFrom(
(
fromEvent(networkManager, eventName as unknown as string) as Observable<T>
).pipe(
filterAsync(async http => {
if (isString(urlOrPredicate)) {
return urlOrPredicate === http.url();
}
if (typeof urlOrPredicate === 'function') {
return !!(await urlOrPredicate(http));
}
return false;
}),
raceWith(timeout(ms), from(cancelation.valueOrThrow()))
)
);
}