thunderstrike/puppeteer
thunderstrike/puppeteer
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor: extract NetworkEventManager from NetworkManager

Browse Source
This commit is contained in:
Ben Allfree 2021-12-07 07:08:45 -08:00 committed by GitHub
parent dc23b7535c
commit bbfad6802c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 197 additions and 148 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

107
src/common/NetworkEventManager.ts Normal file
View File

@ -0,0 +1,107 @@
import { Protocol } from 'devtools-protocol';
import { HTTPRequest } from './HTTPRequest.js';
export type QueuedEvents = {
responseReceived: Protocol.Network.ResponseReceivedEvent;
promise: Promise<void>;
resolver: () => void;
loadingFinished?: Protocol.Network.LoadingFinishedEvent;
loadingFailed?: Protocol.Network.LoadingFailedEvent;
};
export type FetchRequestId = string;
export type NetworkRequestId = string;
export type RedirectInfoMap = Array<{
event: Protocol.Network.RequestWillBeSentEvent;
fetchRequestId?: FetchRequestId;
}>;
/**
* @internal
*
* Helper class to track network events by request ID
*/
export class NetworkEventManager {
/*
* There are four possible orders of events:
* A. `_onRequestWillBeSent`
* B. `_onRequestWillBeSent`, `_onRequestPaused`
* C. `_onRequestPaused`, `_onRequestWillBeSent`
* D. `_onRequestPaused`, `_onRequestWillBeSent`, `_onRequestPaused`,
* `_onRequestWillBeSent`, `_onRequestPaused`, `_onRequestPaused`
* (see crbug.com/1196004)
*
* For `_onRequest` we need the event from `_onRequestWillBeSent` and
* optionally the `interceptionId` from `_onRequestPaused`.
*
* If request interception is disabled, call `_onRequest` once per call to
* `_onRequestWillBeSent`.
* If request interception is enabled, call `_onRequest` once per call to
* `_onRequestPaused` (once per `interceptionId`).
*
* Events are stored to allow for subsequent events to call `_onRequest`.
*
* Note that (chains of) redirect requests have the same `requestId` (!) as
* the original request. We have to anticipate series of events like these:
* A. `_onRequestWillBeSent`,
* `_onRequestWillBeSent`, ...
* B. `_onRequestWillBeSent`, `_onRequestPaused`,
* `_onRequestWillBeSent`, `_onRequestPaused`, ...
* C. `_onRequestWillBeSent`, `_onRequestPaused`,
* `_onRequestPaused`, `_onRequestWillBeSent`, ...
* D. `_onRequestPaused`, `_onRequestWillBeSent`,
* `_onRequestPaused`, `_onRequestWillBeSent`, `_onRequestPaused`,
* `_onRequestWillBeSent`, `_onRequestPaused`, `_onRequestPaused`, ...
* (see crbug.com/1196004)
*/
requestWillBeSent = new Map<
NetworkRequestId,
Protocol.Network.RequestWillBeSentEvent
>();
requestPaused = new Map<
NetworkRequestId,
Protocol.Fetch.RequestPausedEvent
>();
httpRequest = new Map<NetworkRequestId, HTTPRequest>();
/*
* The below maps are used to reconcile Network.responseReceivedExtraInfo
* events with their corresponding request. Each response and redirect
* response gets an ExtraInfo event, and we don't know which will come first.
* This means that we have to store a Response or an ExtraInfo for each
* response, and emit the event when we get both of them. In addition, to
* handle redirects, we have to make them Arrays to represent the chain of
* events.
*/
private _responseReceivedExtraInfo = new Map<
NetworkRequestId,
Protocol.Network.ResponseReceivedExtraInfoEvent[]
>();
private _queuedRedirectInfoMap = new Map<NetworkRequestId, RedirectInfoMap>();
queuedEvents = new Map<NetworkRequestId, QueuedEvents>();
forget(networkRequestId: NetworkRequestId): void {
this.requestWillBeSent.delete(networkRequestId);
this.requestPaused.delete(networkRequestId);
this.queuedEvents.delete(networkRequestId);
this._queuedRedirectInfoMap.delete(networkRequestId);
this._responseReceivedExtraInfo.delete(networkRequestId);
}
responseExtraInfo(
networkRequestId: NetworkRequestId
): Protocol.Network.ResponseReceivedExtraInfoEvent[] {
if (!this._responseReceivedExtraInfo.has(networkRequestId)) {
this._responseReceivedExtraInfo.set(networkRequestId, []);
}
return this._responseReceivedExtraInfo.get(networkRequestId);
}
queuedRedirectInfo(fetchRequestId: FetchRequestId): RedirectInfoMap {
if (!this._queuedRedirectInfoMap.has(fetchRequestId)) {
this._queuedRedirectInfoMap.set(fetchRequestId, []);
}
return this._queuedRedirectInfoMap.get(fetchRequestId);
}
}

238
src/common/NetworkManager.ts
View File

@ -22,6 +22,11 @@ import { helper, debugError } from './helper.js';
import { Protocol } from 'devtools-protocol';
import { HTTPRequest } from './HTTPRequest.js';
import { HTTPResponse } from './HTTPResponse.js';
import {
FetchRequestId,
NetworkEventManager,
NetworkRequestId,
} from './NetworkEventManager.js';
/**
* @public
@ -82,76 +87,7 @@ export class NetworkManager extends EventEmitter {
_ignoreHTTPSErrors: boolean;
_frameManager: FrameManager;
/*
* There are four possible orders of events:
* A. `_onRequestWillBeSent`
* B. `_onRequestWillBeSent`, `_onRequestPaused`
* C. `_onRequestPaused`, `_onRequestWillBeSent`
* D. `_onRequestPaused`, `_onRequestWillBeSent`, `_onRequestPaused`
* (see crbug.com/1196004)
*
* For `_onRequest` we need the event from `_onRequestWillBeSent` and
* optionally the `interceptionId` from `_onRequestPaused`.
*
* If request interception is disabled, call `_onRequest` once per call to
* `_onRequestWillBeSent`.
* If request interception is enabled, call `_onRequest` once per call to
* `_onRequestPaused` (once per `interceptionId`).
*
* Events are stored to allow for subsequent events to call `_onRequest`.
*
* Note that (chains of) redirect requests have the same `requestId` (!) as
* the original request. We have to anticipate series of events like these:
* A. `_onRequestWillBeSent`,
* `_onRequestWillBeSent`, ...
* B. `_onRequestWillBeSent`, `_onRequestPaused`,
* `_onRequestWillBeSent`, `_onRequestPaused`, ...
* C. `_onRequestWillBeSent`, `_onRequestPaused`,
* `_onRequestPaused`, `_onRequestWillBeSent`, ...
* D. `_onRequestPaused`, `_onRequestWillBeSent`,
* `_onRequestPaused`, `_onRequestWillBeSent`, `_onRequestPaused`, ...
* (see crbug.com/1196004)
*/
_requestIdToRequestWillBeSentEvent = new Map<
string,
Protocol.Network.RequestWillBeSentEvent
>();
_requestIdToRequestPausedEvent = new Map<
string,
Protocol.Fetch.RequestPausedEvent
>();
_requestIdToRequest = new Map<string, HTTPRequest>();
/*
* The below maps are used to reconcile Network.responseReceivedExtraInfo
* events with their corresponding request. Each response and redirect
* response gets an ExtraInfo event, and we don't know which will come first.
* This means that we have to store a Response or an ExtraInfo for each
* response, and emit the event when we get both of them. In addition, to
* handle redirects, we have to make them Arrays to represent the chain of
* events.
*/
_requestIdToResponseReceivedExtraInfo = new Map<
string,
Protocol.Network.ResponseReceivedExtraInfoEvent[]
>();
_requestIdToQueuedRedirectInfoMap = new Map<
string,
Array<{
event: Protocol.Network.RequestWillBeSentEvent;
interceptionId?: string;
}>
>();
_requestIdToQueuedEvents = new Map<
string,
{
responseReceived: Protocol.Network.ResponseReceivedEvent;
promise: Promise<void>;
resolver: () => void;
loadingFinished?: Protocol.Network.LoadingFinishedEvent;
loadingFailed?: Protocol.Network.LoadingFailedEvent;
}
>();
_networkEventManager = new NetworkEventManager();
_extraHTTPHeaders: Record<string, string> = {};
_credentials?: Credentials = null;
@ -236,7 +172,7 @@ export class NetworkManager extends EventEmitter {
}
numRequestsInProgress(): number {
return [...this._requestIdToRequest].filter(([, request]) => {
return [...this._networkEventManager.httpRequest].filter(([, request]) => {
return !request.response();
}).length;
}
@ -327,16 +263,19 @@ export class NetworkManager extends EventEmitter {
this._userRequestInterceptionEnabled &&
!event.request.url.startsWith('data:')
) {
const requestId = event.requestId;
const { requestId: networkRequestId } = event;
this._networkEventManager.requestWillBeSent.set(networkRequestId, event);
/**
* CDP may have sent a Fetch.requestPaused event already. Check for it.
*/
const requestPausedEvent =
this._requestIdToRequestPausedEvent.get(requestId);
this._requestIdToRequestWillBeSentEvent.set(requestId, event);
this._networkEventManager.requestPaused.get(networkRequestId);
if (requestPausedEvent) {
const interceptionId = requestPausedEvent.requestId;
this._onRequest(event, interceptionId);
this._requestIdToRequestPausedEvent.delete(requestId);
const { requestId: fetchRequestId } = requestPausedEvent;
this._onRequest(event, fetchRequestId);
this._networkEventManager.requestPaused.delete(networkRequestId);
}
return;
@ -368,6 +307,15 @@ export class NetworkManager extends EventEmitter {
.catch(debugError);
}
/**
* CDP may send a Fetch.requestPaused without or before a
* Network.requestWillBeSent
*
* CDP may send multiple Fetch.requestPaused
* for the same Network.requestWillBeSent.
*
*
*/
_onRequestPaused(event: Protocol.Fetch.RequestPausedEvent): void {
if (
!this._userRequestInterceptionEnabled &&
@ -380,55 +328,38 @@ export class NetworkManager extends EventEmitter {
.catch(debugError);
}
const requestId = event.networkId;
const interceptionId = event.requestId;
const { networkId: networkRequestId, requestId: fetchRequestId } = event;
if (!requestId) {
if (!networkRequestId) {
return;
}
let requestWillBeSentEvent =
this._requestIdToRequestWillBeSentEvent.get(requestId);
const requestWillBeSentEvent = (() => {
const requestWillBeSentEvent =
this._networkEventManager.requestWillBeSent.get(networkRequestId);
// redirect requests have the same `requestId`,
if (
requestWillBeSentEvent &&
(requestWillBeSentEvent.request.url !== event.request.url ||
requestWillBeSentEvent.request.method !== event.request.method)
) {
this._requestIdToRequestWillBeSentEvent.delete(requestId);
requestWillBeSentEvent = null;
}
// redirect requests have the same `requestId`,
if (
requestWillBeSentEvent &&
(requestWillBeSentEvent.request.url !== event.request.url ||
requestWillBeSentEvent.request.method !== event.request.method)
) {
this._networkEventManager.requestWillBeSent.delete(networkRequestId);
return;
}
return requestWillBeSentEvent;
})();
if (requestWillBeSentEvent) {
this._onRequest(requestWillBeSentEvent, interceptionId);
this._onRequest(requestWillBeSentEvent, fetchRequestId);
} else {
this._requestIdToRequestPausedEvent.set(requestId, event);
this._networkEventManager.requestPaused.set(networkRequestId, event);
}
}
_requestIdToQueuedRedirectInfo(requestId: string): Array<{
event: Protocol.Network.RequestWillBeSentEvent;
interceptionId?: string;
}> {
if (!this._requestIdToQueuedRedirectInfoMap.has(requestId)) {
this._requestIdToQueuedRedirectInfoMap.set(requestId, []);
}
return this._requestIdToQueuedRedirectInfoMap.get(requestId);
}
_requestIdToResponseExtraInfo(
requestId: string
): Protocol.Network.ResponseReceivedExtraInfoEvent[] {
if (!this._requestIdToResponseReceivedExtraInfo.has(requestId)) {
this._requestIdToResponseReceivedExtraInfo.set(requestId, []);
}
return this._requestIdToResponseReceivedExtraInfo.get(requestId);
}
_onRequest(
event: Protocol.Network.RequestWillBeSentEvent,
interceptionId?: string
fetchRequestId?: FetchRequestId
): void {
let redirectChain = [];
if (event.redirectResponse) {
@ -441,19 +372,21 @@ export class NetworkManager extends EventEmitter {
// response/requestfinished.
let redirectResponseExtraInfo = null;
if (event.redirectHasExtraInfo) {
redirectResponseExtraInfo = this._requestIdToResponseExtraInfo(
event.requestId
).shift();
redirectResponseExtraInfo = this._networkEventManager
.responseExtraInfo(event.requestId)
.shift();
if (!redirectResponseExtraInfo) {
this._requestIdToQueuedRedirectInfo(event.requestId).push({
this._networkEventManager.queuedRedirectInfo(event.requestId).push({
event,
interceptionId,
fetchRequestId,
});
return;
}
}
const request = this._requestIdToRequest.get(event.requestId);
const request = this._networkEventManager.httpRequest.get(
event.requestId
);
// If we connect late to the target, we could have missed the
// requestWillBeSent event.
if (request) {
@ -471,12 +404,12 @@ export class NetworkManager extends EventEmitter {
const request = new HTTPRequest(
this._client,
frame,
interceptionId,
fetchRequestId,
this._userRequestInterceptionEnabled,
event,
redirectChain
);
this._requestIdToRequest.set(event.requestId, request);
this._networkEventManager.httpRequest.set(event.requestId, request);
this.emit(NetworkManagerEmittedEvents.Request, request);
request.finalizeInterceptions();
}
@ -484,7 +417,7 @@ export class NetworkManager extends EventEmitter {
_onRequestServedFromCache(
event: Protocol.Network.RequestServedFromCacheEvent
): void {
const request = this._requestIdToRequest.get(event.requestId);
const request = this._networkEventManager.httpRequest.get(event.requestId);
if (request) request._fromMemoryCache = true;
this.emit(NetworkManagerEmittedEvents.RequestServedFromCache, request);
}
@ -514,11 +447,13 @@ export class NetworkManager extends EventEmitter {
responseReceived: Protocol.Network.ResponseReceivedEvent,
extraInfo: Protocol.Network.ResponseReceivedExtraInfoEvent | null
): void {
const request = this._requestIdToRequest.get(responseReceived.requestId);
const request = this._networkEventManager.httpRequest.get(
responseReceived.requestId
);
// FileUpload sends a response without a matching request.
if (!request) return;
const extraInfos = this._requestIdToResponseExtraInfo(
const extraInfos = this._networkEventManager.responseExtraInfo(
responseReceived.requestId
);
if (extraInfos.length) {
@ -541,15 +476,17 @@ export class NetworkManager extends EventEmitter {
}
_onResponseReceived(event: Protocol.Network.ResponseReceivedEvent): void {
const request = this._requestIdToRequest.get(event.requestId);
const request = this._networkEventManager.httpRequest.get(event.requestId);
let extraInfo = null;
if (request && !request._fromMemoryCache && event.hasExtraInfo) {
extraInfo = this._requestIdToResponseExtraInfo(event.requestId).shift();
extraInfo = this._networkEventManager
.responseExtraInfo(event.requestId)
.shift();
if (!extraInfo) {
// Wait until we get the corresponding ExtraInfo event.
let resolver = null;
const promise = new Promise<void>((resolve) => (resolver = resolve));
this._requestIdToQueuedEvents.set(event.requestId, {
this._networkEventManager.queuedEvents.set(event.requestId, {
responseReceived: event,
promise,
resolver,
@ -560,8 +497,11 @@ export class NetworkManager extends EventEmitter {
this._emitResponseEvent(event, extraInfo);
}
responseWaitingForExtraInfoPromise(requestId: string): Promise<void> {
const responseReceived = this._requestIdToQueuedEvents.get(requestId);
responseWaitingForExtraInfoPromise(
networkRequestId: NetworkRequestId
): Promise<void> {
const responseReceived =
this._networkEventManager.queuedEvents.get(networkRequestId);
if (!responseReceived) return Promise.resolve();
return responseReceived.promise;
}
@ -572,18 +512,20 @@ export class NetworkManager extends EventEmitter {
// We may have skipped a redirect response/request pair due to waiting for
// this ExtraInfo event. If so, continue that work now that we have the
// request.
const redirectInfo = this._requestIdToQueuedRedirectInfo(
event.requestId
).shift();
const redirectInfo = this._networkEventManager
.queuedRedirectInfo(event.requestId)
.shift();
if (redirectInfo) {
this._requestIdToResponseExtraInfo(event.requestId).push(event);
this._onRequest(redirectInfo.event, redirectInfo.interceptionId);
this._networkEventManager.responseExtraInfo(event.requestId).push(event);
this._onRequest(redirectInfo.event, redirectInfo.fetchRequestId);
return;
}
// We may have skipped response and loading events because we didn't have
// this ExtraInfo event yet. If so, emit those events now.
const queuedEvents = this._requestIdToQueuedEvents.get(event.requestId);
const queuedEvents = this._networkEventManager.queuedEvents.get(
event.requestId
);
if (queuedEvents) {
this._emitResponseEvent(queuedEvents.responseReceived, event);
if (queuedEvents.loadingFinished) {
@ -597,29 +539,27 @@ export class NetworkManager extends EventEmitter {
}
// Wait until we get another event that can use this ExtraInfo event.
this._requestIdToResponseExtraInfo(event.requestId).push(event);
this._networkEventManager.responseExtraInfo(event.requestId).push(event);
}
_forgetRequest(request: HTTPRequest, events: boolean): void {
const requestId = request._requestId;
const interceptionId = request._interceptionId;
this._requestIdToRequest.delete(requestId);
this._networkEventManager.httpRequest.delete(requestId);
this._attemptedAuthentications.delete(interceptionId);
if (events) {
this._requestIdToRequestWillBeSentEvent.delete(requestId);
this._requestIdToRequestPausedEvent.delete(requestId);
this._requestIdToQueuedEvents.delete(requestId);
this._requestIdToQueuedRedirectInfoMap.delete(requestId);
this._requestIdToResponseReceivedExtraInfo.delete(requestId);
this._networkEventManager.forget(requestId);
}
}
_onLoadingFinished(event: Protocol.Network.LoadingFinishedEvent): void {
// If the response event for this request is still waiting on a
// corresponding ExtraInfo event, then wait to emit this event too.
const queuedEvents = this._requestIdToQueuedEvents.get(event.requestId);
const queuedEvents = this._networkEventManager.queuedEvents.get(
event.requestId
);
if (queuedEvents) {
queuedEvents.loadingFinished = event;
} else {
@ -628,7 +568,7 @@ export class NetworkManager extends EventEmitter {
}
_emitLoadingFinished(event: Protocol.Network.LoadingFinishedEvent): void {
const request = this._requestIdToRequest.get(event.requestId);
const request = this._networkEventManager.httpRequest.get(event.requestId);
// For certain requestIds we never receive requestWillBeSent event.
// @see https://crbug.com/750469
if (!request) return;
@ -643,7 +583,9 @@ export class NetworkManager extends EventEmitter {
_onLoadingFailed(event: Protocol.Network.LoadingFailedEvent): void {
// If the response event for this request is still waiting on a
// corresponding ExtraInfo event, then wait to emit this event too.
const queuedEvents = this._requestIdToQueuedEvents.get(event.requestId);
const queuedEvents = this._networkEventManager.queuedEvents.get(
event.requestId
);
if (queuedEvents) {
queuedEvents.loadingFailed = event;
} else {
@ -652,7 +594,7 @@ export class NetworkManager extends EventEmitter {
}
_emitLoadingFailed(event: Protocol.Network.LoadingFailedEvent): void {
const request = this._requestIdToRequest.get(event.requestId);
const request = this._networkEventManager.httpRequest.get(event.requestId);
// For certain requestIds we never receive requestWillBeSent event.
// @see https://crbug.com/750469
if (!request) return;