diff --git a/src/common/NetworkEventManager.ts b/src/common/NetworkEventManager.ts new file mode 100644 index 00000000..2ac263ca --- /dev/null +++ b/src/common/NetworkEventManager.ts @@ -0,0 +1,107 @@ +import { Protocol } from 'devtools-protocol'; +import { HTTPRequest } from './HTTPRequest.js'; + +export type QueuedEvents = { + responseReceived: Protocol.Network.ResponseReceivedEvent; + promise: Promise; + resolver: () => void; + loadingFinished?: Protocol.Network.LoadingFinishedEvent; + loadingFailed?: Protocol.Network.LoadingFailedEvent; +}; + +export type FetchRequestId = string; +export type NetworkRequestId = string; + +export type RedirectInfoMap = Array<{ + event: Protocol.Network.RequestWillBeSentEvent; + fetchRequestId?: FetchRequestId; +}>; + +/** + * @internal + * + * Helper class to track network events by request ID + */ +export class NetworkEventManager { + /* + * There are four possible orders of events: + * A. `_onRequestWillBeSent` + * B. `_onRequestWillBeSent`, `_onRequestPaused` + * C. `_onRequestPaused`, `_onRequestWillBeSent` + * D. `_onRequestPaused`, `_onRequestWillBeSent`, `_onRequestPaused`, + * `_onRequestWillBeSent`, `_onRequestPaused`, `_onRequestPaused` + * (see crbug.com/1196004) + * + * For `_onRequest` we need the event from `_onRequestWillBeSent` and + * optionally the `interceptionId` from `_onRequestPaused`. + * + * If request interception is disabled, call `_onRequest` once per call to + * `_onRequestWillBeSent`. + * If request interception is enabled, call `_onRequest` once per call to + * `_onRequestPaused` (once per `interceptionId`). + * + * Events are stored to allow for subsequent events to call `_onRequest`. + * + * Note that (chains of) redirect requests have the same `requestId` (!) as + * the original request. We have to anticipate series of events like these: + * A. `_onRequestWillBeSent`, + * `_onRequestWillBeSent`, ... + * B. `_onRequestWillBeSent`, `_onRequestPaused`, + * `_onRequestWillBeSent`, `_onRequestPaused`, ... + * C. `_onRequestWillBeSent`, `_onRequestPaused`, + * `_onRequestPaused`, `_onRequestWillBeSent`, ... + * D. `_onRequestPaused`, `_onRequestWillBeSent`, + * `_onRequestPaused`, `_onRequestWillBeSent`, `_onRequestPaused`, + * `_onRequestWillBeSent`, `_onRequestPaused`, `_onRequestPaused`, ... + * (see crbug.com/1196004) + */ + requestWillBeSent = new Map< + NetworkRequestId, + Protocol.Network.RequestWillBeSentEvent + >(); + requestPaused = new Map< + NetworkRequestId, + Protocol.Fetch.RequestPausedEvent + >(); + httpRequest = new Map(); + + /* + * The below maps are used to reconcile Network.responseReceivedExtraInfo + * events with their corresponding request. Each response and redirect + * response gets an ExtraInfo event, and we don't know which will come first. + * This means that we have to store a Response or an ExtraInfo for each + * response, and emit the event when we get both of them. In addition, to + * handle redirects, we have to make them Arrays to represent the chain of + * events. + */ + private _responseReceivedExtraInfo = new Map< + NetworkRequestId, + Protocol.Network.ResponseReceivedExtraInfoEvent[] + >(); + private _queuedRedirectInfoMap = new Map(); + queuedEvents = new Map(); + + forget(networkRequestId: NetworkRequestId): void { + this.requestWillBeSent.delete(networkRequestId); + this.requestPaused.delete(networkRequestId); + this.queuedEvents.delete(networkRequestId); + this._queuedRedirectInfoMap.delete(networkRequestId); + this._responseReceivedExtraInfo.delete(networkRequestId); + } + + responseExtraInfo( + networkRequestId: NetworkRequestId + ): Protocol.Network.ResponseReceivedExtraInfoEvent[] { + if (!this._responseReceivedExtraInfo.has(networkRequestId)) { + this._responseReceivedExtraInfo.set(networkRequestId, []); + } + return this._responseReceivedExtraInfo.get(networkRequestId); + } + + queuedRedirectInfo(fetchRequestId: FetchRequestId): RedirectInfoMap { + if (!this._queuedRedirectInfoMap.has(fetchRequestId)) { + this._queuedRedirectInfoMap.set(fetchRequestId, []); + } + return this._queuedRedirectInfoMap.get(fetchRequestId); + } +} diff --git a/src/common/NetworkManager.ts b/src/common/NetworkManager.ts index 5c755bc3..185d8805 100644 --- a/src/common/NetworkManager.ts +++ b/src/common/NetworkManager.ts @@ -22,6 +22,11 @@ import { helper, debugError } from './helper.js'; import { Protocol } from 'devtools-protocol'; import { HTTPRequest } from './HTTPRequest.js'; import { HTTPResponse } from './HTTPResponse.js'; +import { + FetchRequestId, + NetworkEventManager, + NetworkRequestId, +} from './NetworkEventManager.js'; /** * @public @@ -82,76 +87,7 @@ export class NetworkManager extends EventEmitter { _ignoreHTTPSErrors: boolean; _frameManager: FrameManager; - /* - * There are four possible orders of events: - * A. `_onRequestWillBeSent` - * B. `_onRequestWillBeSent`, `_onRequestPaused` - * C. `_onRequestPaused`, `_onRequestWillBeSent` - * D. `_onRequestPaused`, `_onRequestWillBeSent`, `_onRequestPaused` - * (see crbug.com/1196004) - * - * For `_onRequest` we need the event from `_onRequestWillBeSent` and - * optionally the `interceptionId` from `_onRequestPaused`. - * - * If request interception is disabled, call `_onRequest` once per call to - * `_onRequestWillBeSent`. - * If request interception is enabled, call `_onRequest` once per call to - * `_onRequestPaused` (once per `interceptionId`). - * - * Events are stored to allow for subsequent events to call `_onRequest`. - * - * Note that (chains of) redirect requests have the same `requestId` (!) as - * the original request. We have to anticipate series of events like these: - * A. `_onRequestWillBeSent`, - * `_onRequestWillBeSent`, ... - * B. `_onRequestWillBeSent`, `_onRequestPaused`, - * `_onRequestWillBeSent`, `_onRequestPaused`, ... - * C. `_onRequestWillBeSent`, `_onRequestPaused`, - * `_onRequestPaused`, `_onRequestWillBeSent`, ... - * D. `_onRequestPaused`, `_onRequestWillBeSent`, - * `_onRequestPaused`, `_onRequestWillBeSent`, `_onRequestPaused`, ... - * (see crbug.com/1196004) - */ - _requestIdToRequestWillBeSentEvent = new Map< - string, - Protocol.Network.RequestWillBeSentEvent - >(); - _requestIdToRequestPausedEvent = new Map< - string, - Protocol.Fetch.RequestPausedEvent - >(); - _requestIdToRequest = new Map(); - - /* - * The below maps are used to reconcile Network.responseReceivedExtraInfo - * events with their corresponding request. Each response and redirect - * response gets an ExtraInfo event, and we don't know which will come first. - * This means that we have to store a Response or an ExtraInfo for each - * response, and emit the event when we get both of them. In addition, to - * handle redirects, we have to make them Arrays to represent the chain of - * events. - */ - _requestIdToResponseReceivedExtraInfo = new Map< - string, - Protocol.Network.ResponseReceivedExtraInfoEvent[] - >(); - _requestIdToQueuedRedirectInfoMap = new Map< - string, - Array<{ - event: Protocol.Network.RequestWillBeSentEvent; - interceptionId?: string; - }> - >(); - _requestIdToQueuedEvents = new Map< - string, - { - responseReceived: Protocol.Network.ResponseReceivedEvent; - promise: Promise; - resolver: () => void; - loadingFinished?: Protocol.Network.LoadingFinishedEvent; - loadingFailed?: Protocol.Network.LoadingFailedEvent; - } - >(); + _networkEventManager = new NetworkEventManager(); _extraHTTPHeaders: Record = {}; _credentials?: Credentials = null; @@ -236,7 +172,7 @@ export class NetworkManager extends EventEmitter { } numRequestsInProgress(): number { - return [...this._requestIdToRequest].filter(([, request]) => { + return [...this._networkEventManager.httpRequest].filter(([, request]) => { return !request.response(); }).length; } @@ -327,16 +263,19 @@ export class NetworkManager extends EventEmitter { this._userRequestInterceptionEnabled && !event.request.url.startsWith('data:') ) { - const requestId = event.requestId; + const { requestId: networkRequestId } = event; + + this._networkEventManager.requestWillBeSent.set(networkRequestId, event); + + /** + * CDP may have sent a Fetch.requestPaused event already. Check for it. + */ const requestPausedEvent = - this._requestIdToRequestPausedEvent.get(requestId); - - this._requestIdToRequestWillBeSentEvent.set(requestId, event); - + this._networkEventManager.requestPaused.get(networkRequestId); if (requestPausedEvent) { - const interceptionId = requestPausedEvent.requestId; - this._onRequest(event, interceptionId); - this._requestIdToRequestPausedEvent.delete(requestId); + const { requestId: fetchRequestId } = requestPausedEvent; + this._onRequest(event, fetchRequestId); + this._networkEventManager.requestPaused.delete(networkRequestId); } return; @@ -368,6 +307,15 @@ export class NetworkManager extends EventEmitter { .catch(debugError); } + /** + * CDP may send a Fetch.requestPaused without or before a + * Network.requestWillBeSent + * + * CDP may send multiple Fetch.requestPaused + * for the same Network.requestWillBeSent. + * + * + */ _onRequestPaused(event: Protocol.Fetch.RequestPausedEvent): void { if ( !this._userRequestInterceptionEnabled && @@ -380,55 +328,38 @@ export class NetworkManager extends EventEmitter { .catch(debugError); } - const requestId = event.networkId; - const interceptionId = event.requestId; + const { networkId: networkRequestId, requestId: fetchRequestId } = event; - if (!requestId) { + if (!networkRequestId) { return; } - let requestWillBeSentEvent = - this._requestIdToRequestWillBeSentEvent.get(requestId); + const requestWillBeSentEvent = (() => { + const requestWillBeSentEvent = + this._networkEventManager.requestWillBeSent.get(networkRequestId); - // redirect requests have the same `requestId`, - if ( - requestWillBeSentEvent && - (requestWillBeSentEvent.request.url !== event.request.url || - requestWillBeSentEvent.request.method !== event.request.method) - ) { - this._requestIdToRequestWillBeSentEvent.delete(requestId); - requestWillBeSentEvent = null; - } + // redirect requests have the same `requestId`, + if ( + requestWillBeSentEvent && + (requestWillBeSentEvent.request.url !== event.request.url || + requestWillBeSentEvent.request.method !== event.request.method) + ) { + this._networkEventManager.requestWillBeSent.delete(networkRequestId); + return; + } + return requestWillBeSentEvent; + })(); if (requestWillBeSentEvent) { - this._onRequest(requestWillBeSentEvent, interceptionId); + this._onRequest(requestWillBeSentEvent, fetchRequestId); } else { - this._requestIdToRequestPausedEvent.set(requestId, event); + this._networkEventManager.requestPaused.set(networkRequestId, event); } } - _requestIdToQueuedRedirectInfo(requestId: string): Array<{ - event: Protocol.Network.RequestWillBeSentEvent; - interceptionId?: string; - }> { - if (!this._requestIdToQueuedRedirectInfoMap.has(requestId)) { - this._requestIdToQueuedRedirectInfoMap.set(requestId, []); - } - return this._requestIdToQueuedRedirectInfoMap.get(requestId); - } - - _requestIdToResponseExtraInfo( - requestId: string - ): Protocol.Network.ResponseReceivedExtraInfoEvent[] { - if (!this._requestIdToResponseReceivedExtraInfo.has(requestId)) { - this._requestIdToResponseReceivedExtraInfo.set(requestId, []); - } - return this._requestIdToResponseReceivedExtraInfo.get(requestId); - } - _onRequest( event: Protocol.Network.RequestWillBeSentEvent, - interceptionId?: string + fetchRequestId?: FetchRequestId ): void { let redirectChain = []; if (event.redirectResponse) { @@ -441,19 +372,21 @@ export class NetworkManager extends EventEmitter { // response/requestfinished. let redirectResponseExtraInfo = null; if (event.redirectHasExtraInfo) { - redirectResponseExtraInfo = this._requestIdToResponseExtraInfo( - event.requestId - ).shift(); + redirectResponseExtraInfo = this._networkEventManager + .responseExtraInfo(event.requestId) + .shift(); if (!redirectResponseExtraInfo) { - this._requestIdToQueuedRedirectInfo(event.requestId).push({ + this._networkEventManager.queuedRedirectInfo(event.requestId).push({ event, - interceptionId, + fetchRequestId, }); return; } } - const request = this._requestIdToRequest.get(event.requestId); + const request = this._networkEventManager.httpRequest.get( + event.requestId + ); // If we connect late to the target, we could have missed the // requestWillBeSent event. if (request) { @@ -471,12 +404,12 @@ export class NetworkManager extends EventEmitter { const request = new HTTPRequest( this._client, frame, - interceptionId, + fetchRequestId, this._userRequestInterceptionEnabled, event, redirectChain ); - this._requestIdToRequest.set(event.requestId, request); + this._networkEventManager.httpRequest.set(event.requestId, request); this.emit(NetworkManagerEmittedEvents.Request, request); request.finalizeInterceptions(); } @@ -484,7 +417,7 @@ export class NetworkManager extends EventEmitter { _onRequestServedFromCache( event: Protocol.Network.RequestServedFromCacheEvent ): void { - const request = this._requestIdToRequest.get(event.requestId); + const request = this._networkEventManager.httpRequest.get(event.requestId); if (request) request._fromMemoryCache = true; this.emit(NetworkManagerEmittedEvents.RequestServedFromCache, request); } @@ -514,11 +447,13 @@ export class NetworkManager extends EventEmitter { responseReceived: Protocol.Network.ResponseReceivedEvent, extraInfo: Protocol.Network.ResponseReceivedExtraInfoEvent | null ): void { - const request = this._requestIdToRequest.get(responseReceived.requestId); + const request = this._networkEventManager.httpRequest.get( + responseReceived.requestId + ); // FileUpload sends a response without a matching request. if (!request) return; - const extraInfos = this._requestIdToResponseExtraInfo( + const extraInfos = this._networkEventManager.responseExtraInfo( responseReceived.requestId ); if (extraInfos.length) { @@ -541,15 +476,17 @@ export class NetworkManager extends EventEmitter { } _onResponseReceived(event: Protocol.Network.ResponseReceivedEvent): void { - const request = this._requestIdToRequest.get(event.requestId); + const request = this._networkEventManager.httpRequest.get(event.requestId); let extraInfo = null; if (request && !request._fromMemoryCache && event.hasExtraInfo) { - extraInfo = this._requestIdToResponseExtraInfo(event.requestId).shift(); + extraInfo = this._networkEventManager + .responseExtraInfo(event.requestId) + .shift(); if (!extraInfo) { // Wait until we get the corresponding ExtraInfo event. let resolver = null; const promise = new Promise((resolve) => (resolver = resolve)); - this._requestIdToQueuedEvents.set(event.requestId, { + this._networkEventManager.queuedEvents.set(event.requestId, { responseReceived: event, promise, resolver, @@ -560,8 +497,11 @@ export class NetworkManager extends EventEmitter { this._emitResponseEvent(event, extraInfo); } - responseWaitingForExtraInfoPromise(requestId: string): Promise { - const responseReceived = this._requestIdToQueuedEvents.get(requestId); + responseWaitingForExtraInfoPromise( + networkRequestId: NetworkRequestId + ): Promise { + const responseReceived = + this._networkEventManager.queuedEvents.get(networkRequestId); if (!responseReceived) return Promise.resolve(); return responseReceived.promise; } @@ -572,18 +512,20 @@ export class NetworkManager extends EventEmitter { // We may have skipped a redirect response/request pair due to waiting for // this ExtraInfo event. If so, continue that work now that we have the // request. - const redirectInfo = this._requestIdToQueuedRedirectInfo( - event.requestId - ).shift(); + const redirectInfo = this._networkEventManager + .queuedRedirectInfo(event.requestId) + .shift(); if (redirectInfo) { - this._requestIdToResponseExtraInfo(event.requestId).push(event); - this._onRequest(redirectInfo.event, redirectInfo.interceptionId); + this._networkEventManager.responseExtraInfo(event.requestId).push(event); + this._onRequest(redirectInfo.event, redirectInfo.fetchRequestId); return; } // We may have skipped response and loading events because we didn't have // this ExtraInfo event yet. If so, emit those events now. - const queuedEvents = this._requestIdToQueuedEvents.get(event.requestId); + const queuedEvents = this._networkEventManager.queuedEvents.get( + event.requestId + ); if (queuedEvents) { this._emitResponseEvent(queuedEvents.responseReceived, event); if (queuedEvents.loadingFinished) { @@ -597,29 +539,27 @@ export class NetworkManager extends EventEmitter { } // Wait until we get another event that can use this ExtraInfo event. - this._requestIdToResponseExtraInfo(event.requestId).push(event); + this._networkEventManager.responseExtraInfo(event.requestId).push(event); } _forgetRequest(request: HTTPRequest, events: boolean): void { const requestId = request._requestId; const interceptionId = request._interceptionId; - this._requestIdToRequest.delete(requestId); + this._networkEventManager.httpRequest.delete(requestId); this._attemptedAuthentications.delete(interceptionId); if (events) { - this._requestIdToRequestWillBeSentEvent.delete(requestId); - this._requestIdToRequestPausedEvent.delete(requestId); - this._requestIdToQueuedEvents.delete(requestId); - this._requestIdToQueuedRedirectInfoMap.delete(requestId); - this._requestIdToResponseReceivedExtraInfo.delete(requestId); + this._networkEventManager.forget(requestId); } } _onLoadingFinished(event: Protocol.Network.LoadingFinishedEvent): void { // If the response event for this request is still waiting on a // corresponding ExtraInfo event, then wait to emit this event too. - const queuedEvents = this._requestIdToQueuedEvents.get(event.requestId); + const queuedEvents = this._networkEventManager.queuedEvents.get( + event.requestId + ); if (queuedEvents) { queuedEvents.loadingFinished = event; } else { @@ -628,7 +568,7 @@ export class NetworkManager extends EventEmitter { } _emitLoadingFinished(event: Protocol.Network.LoadingFinishedEvent): void { - const request = this._requestIdToRequest.get(event.requestId); + const request = this._networkEventManager.httpRequest.get(event.requestId); // For certain requestIds we never receive requestWillBeSent event. // @see https://crbug.com/750469 if (!request) return; @@ -643,7 +583,9 @@ export class NetworkManager extends EventEmitter { _onLoadingFailed(event: Protocol.Network.LoadingFailedEvent): void { // If the response event for this request is still waiting on a // corresponding ExtraInfo event, then wait to emit this event too. - const queuedEvents = this._requestIdToQueuedEvents.get(event.requestId); + const queuedEvents = this._networkEventManager.queuedEvents.get( + event.requestId + ); if (queuedEvents) { queuedEvents.loadingFailed = event; } else { @@ -652,7 +594,7 @@ export class NetworkManager extends EventEmitter { } _emitLoadingFailed(event: Protocol.Network.LoadingFailedEvent): void { - const request = this._requestIdToRequest.get(event.requestId); + const request = this._networkEventManager.httpRequest.get(event.requestId); // For certain requestIds we never receive requestWillBeSent event. // @see https://crbug.com/750469 if (!request) return;