puppeteer/test/NetworkManager.spec.ts

664 lines
23 KiB
TypeScript
Raw Normal View History

/**
* Copyright 2020 Google Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { describeChromeOnly } from './mocha-utils'; // eslint-disable-line import/extensions
import expect from 'expect';
import {
NetworkManager,
NetworkManagerEmittedEvents,
} from '../lib/cjs/puppeteer/common/NetworkManager.js';
import { HTTPRequest } from '../lib/cjs/puppeteer/common/HTTPRequest.js';
import { EventEmitter } from '../lib/cjs/puppeteer/common/EventEmitter.js';
import { Frame } from '../lib/cjs/puppeteer/common/FrameManager.js';
class MockCDPSession extends EventEmitter {
async send(): Promise<any> {}
}
describeChromeOnly('NetworkManager', () => {
it('should process extra info on multiple redirects', async () => {
const mockCDPSession = new MockCDPSession();
new NetworkManager(mockCDPSession, true, {
frame(): Frame | null {
return null;
},
});
mockCDPSession.emit('Network.requestWillBeSent', {
requestId: '7760711DEFCFA23132D98ABA6B4E175C',
loaderId: '7760711DEFCFA23132D98ABA6B4E175C',
documentURL: 'http://localhost:8907/redirect/1.html',
request: {
url: 'http://localhost:8907/redirect/1.html',
method: 'GET',
headers: {
'Upgrade-Insecure-Requests': '1',
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/97.0.4691.0 Safari/537.36',
},
mixedContentType: 'none',
initialPriority: 'VeryHigh',
referrerPolicy: 'strict-origin-when-cross-origin',
isSameSite: true,
},
timestamp: 2111.55635,
wallTime: 1637315638.473634,
initiator: { type: 'other' },
redirectHasExtraInfo: false,
type: 'Document',
frameId: '099A5216AF03AAFEC988F214B024DF08',
hasUserGesture: false,
});
mockCDPSession.emit('Network.requestWillBeSentExtraInfo', {
requestId: '7760711DEFCFA23132D98ABA6B4E175C',
associatedCookies: [],
headers: {
Host: 'localhost:8907',
Connection: 'keep-alive',
'Upgrade-Insecure-Requests': '1',
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/97.0.4691.0 Safari/537.36',
Accept:
'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
'Sec-Fetch-Site': 'none',
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-User': '?1',
'Sec-Fetch-Dest': 'document',
'Accept-Encoding': 'gzip, deflate, br',
},
connectTiming: { requestTime: 2111.557593 },
});
mockCDPSession.emit('Network.responseReceivedExtraInfo', {
requestId: '7760711DEFCFA23132D98ABA6B4E175C',
blockedCookies: [],
headers: {
location: '/redirect/2.html',
Date: 'Fri, 19 Nov 2021 09:53:58 GMT',
Connection: 'keep-alive',
'Keep-Alive': 'timeout=5',
'Transfer-Encoding': 'chunked',
},
resourceIPAddressSpace: 'Local',
statusCode: 302,
headersText:
'HTTP/1.1 302 Found\r\nlocation: /redirect/2.html\r\nDate: Fri, 19 Nov 2021 09:53:58 GMT\r\nConnection: keep-alive\r\nKeep-Alive: timeout=5\r\nTransfer-Encoding: chunked\r\n\r\n',
});
mockCDPSession.emit('Network.requestWillBeSent', {
requestId: '7760711DEFCFA23132D98ABA6B4E175C',
loaderId: '7760711DEFCFA23132D98ABA6B4E175C',
documentURL: 'http://localhost:8907/redirect/2.html',
request: {
url: 'http://localhost:8907/redirect/2.html',
method: 'GET',
headers: {
'Upgrade-Insecure-Requests': '1',
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/97.0.4691.0 Safari/537.36',
},
mixedContentType: 'none',
initialPriority: 'VeryHigh',
referrerPolicy: 'strict-origin-when-cross-origin',
isSameSite: true,
},
timestamp: 2111.559124,
wallTime: 1637315638.47642,
initiator: { type: 'other' },
redirectHasExtraInfo: true,
redirectResponse: {
url: 'http://localhost:8907/redirect/1.html',
status: 302,
statusText: 'Found',
headers: {
location: '/redirect/2.html',
Date: 'Fri, 19 Nov 2021 09:53:58 GMT',
Connection: 'keep-alive',
'Keep-Alive': 'timeout=5',
'Transfer-Encoding': 'chunked',
},
mimeType: '',
connectionReused: false,
connectionId: 322,
remoteIPAddress: '[::1]',
remotePort: 8907,
fromDiskCache: false,
fromServiceWorker: false,
fromPrefetchCache: false,
encodedDataLength: 162,
timing: {
requestTime: 2111.557593,
proxyStart: -1,
proxyEnd: -1,
dnsStart: 0.241,
dnsEnd: 0.251,
connectStart: 0.251,
connectEnd: 0.47,
sslStart: -1,
sslEnd: -1,
workerStart: -1,
workerReady: -1,
workerFetchStart: -1,
workerRespondWithSettled: -1,
sendStart: 0.537,
sendEnd: 0.611,
pushStart: 0,
pushEnd: 0,
receiveHeadersEnd: 0.939,
},
responseTime: 1.637315638475744e12,
protocol: 'http/1.1',
securityState: 'secure',
},
type: 'Document',
frameId: '099A5216AF03AAFEC988F214B024DF08',
hasUserGesture: false,
});
mockCDPSession.emit('Network.requestWillBeSentExtraInfo', {
requestId: '7760711DEFCFA23132D98ABA6B4E175C',
associatedCookies: [],
headers: {
Host: 'localhost:8907',
Connection: 'keep-alive',
'Upgrade-Insecure-Requests': '1',
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/97.0.4691.0 Safari/537.36',
Accept:
'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
'Sec-Fetch-Site': 'none',
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-User': '?1',
'Sec-Fetch-Dest': 'document',
'Accept-Encoding': 'gzip, deflate, br',
},
connectTiming: { requestTime: 2111.559346 },
});
mockCDPSession.emit('Network.requestWillBeSent', {
requestId: '7760711DEFCFA23132D98ABA6B4E175C',
loaderId: '7760711DEFCFA23132D98ABA6B4E175C',
documentURL: 'http://localhost:8907/redirect/3.html',
request: {
url: 'http://localhost:8907/redirect/3.html',
method: 'GET',
headers: {
'Upgrade-Insecure-Requests': '1',
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/97.0.4691.0 Safari/537.36',
},
mixedContentType: 'none',
initialPriority: 'VeryHigh',
referrerPolicy: 'strict-origin-when-cross-origin',
isSameSite: true,
},
timestamp: 2111.560249,
wallTime: 1637315638.477543,
initiator: { type: 'other' },
redirectHasExtraInfo: true,
redirectResponse: {
url: 'http://localhost:8907/redirect/2.html',
status: 302,
statusText: 'Found',
headers: {
location: '/redirect/3.html',
Date: 'Fri, 19 Nov 2021 09:53:58 GMT',
Connection: 'keep-alive',
'Keep-Alive': 'timeout=5',
'Transfer-Encoding': 'chunked',
},
mimeType: '',
connectionReused: true,
connectionId: 322,
remoteIPAddress: '[::1]',
remotePort: 8907,
fromDiskCache: false,
fromServiceWorker: false,
fromPrefetchCache: false,
encodedDataLength: 162,
timing: {
requestTime: 2111.559346,
proxyStart: -1,
proxyEnd: -1,
dnsStart: -1,
dnsEnd: -1,
connectStart: -1,
connectEnd: -1,
sslStart: -1,
sslEnd: -1,
workerStart: -1,
workerReady: -1,
workerFetchStart: -1,
workerRespondWithSettled: -1,
sendStart: 0.15,
sendEnd: 0.196,
pushStart: 0,
pushEnd: 0,
receiveHeadersEnd: 0.507,
},
responseTime: 1.637315638477063e12,
protocol: 'http/1.1',
securityState: 'secure',
},
type: 'Document',
frameId: '099A5216AF03AAFEC988F214B024DF08',
hasUserGesture: false,
});
mockCDPSession.emit('Network.responseReceivedExtraInfo', {
requestId: '7760711DEFCFA23132D98ABA6B4E175C',
blockedCookies: [],
headers: {
location: '/redirect/3.html',
Date: 'Fri, 19 Nov 2021 09:53:58 GMT',
Connection: 'keep-alive',
'Keep-Alive': 'timeout=5',
'Transfer-Encoding': 'chunked',
},
resourceIPAddressSpace: 'Local',
statusCode: 302,
headersText:
'HTTP/1.1 302 Found\r\nlocation: /redirect/3.html\r\nDate: Fri, 19 Nov 2021 09:53:58 GMT\r\nConnection: keep-alive\r\nKeep-Alive: timeout=5\r\nTransfer-Encoding: chunked\r\n\r\n',
});
mockCDPSession.emit('Network.requestWillBeSentExtraInfo', {
requestId: '7760711DEFCFA23132D98ABA6B4E175C',
associatedCookies: [],
headers: {
Host: 'localhost:8907',
Connection: 'keep-alive',
'Upgrade-Insecure-Requests': '1',
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/97.0.4691.0 Safari/537.36',
Accept:
'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
'Sec-Fetch-Site': 'none',
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-User': '?1',
'Sec-Fetch-Dest': 'document',
'Accept-Encoding': 'gzip, deflate, br',
},
connectTiming: { requestTime: 2111.560482 },
});
mockCDPSession.emit('Network.requestWillBeSent', {
requestId: '7760711DEFCFA23132D98ABA6B4E175C',
loaderId: '7760711DEFCFA23132D98ABA6B4E175C',
documentURL: 'http://localhost:8907/empty.html',
request: {
url: 'http://localhost:8907/empty.html',
method: 'GET',
headers: {
'Upgrade-Insecure-Requests': '1',
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/97.0.4691.0 Safari/537.36',
},
mixedContentType: 'none',
initialPriority: 'VeryHigh',
referrerPolicy: 'strict-origin-when-cross-origin',
isSameSite: true,
},
timestamp: 2111.561542,
wallTime: 1637315638.478837,
initiator: { type: 'other' },
redirectHasExtraInfo: true,
redirectResponse: {
url: 'http://localhost:8907/redirect/3.html',
status: 302,
statusText: 'Found',
headers: {
location: 'http://localhost:8907/empty.html',
Date: 'Fri, 19 Nov 2021 09:53:58 GMT',
Connection: 'keep-alive',
'Keep-Alive': 'timeout=5',
'Transfer-Encoding': 'chunked',
},
mimeType: '',
connectionReused: true,
connectionId: 322,
remoteIPAddress: '[::1]',
remotePort: 8907,
fromDiskCache: false,
fromServiceWorker: false,
fromPrefetchCache: false,
encodedDataLength: 178,
timing: {
requestTime: 2111.560482,
proxyStart: -1,
proxyEnd: -1,
dnsStart: -1,
dnsEnd: -1,
connectStart: -1,
connectEnd: -1,
sslStart: -1,
sslEnd: -1,
workerStart: -1,
workerReady: -1,
workerFetchStart: -1,
workerRespondWithSettled: -1,
sendStart: 0.149,
sendEnd: 0.198,
pushStart: 0,
pushEnd: 0,
receiveHeadersEnd: 0.478,
},
responseTime: 1.637315638478184e12,
protocol: 'http/1.1',
securityState: 'secure',
},
type: 'Document',
frameId: '099A5216AF03AAFEC988F214B024DF08',
hasUserGesture: false,
});
mockCDPSession.emit('Network.responseReceivedExtraInfo', {
requestId: '7760711DEFCFA23132D98ABA6B4E175C',
blockedCookies: [],
headers: {
location: 'http://localhost:8907/empty.html',
Date: 'Fri, 19 Nov 2021 09:53:58 GMT',
Connection: 'keep-alive',
'Keep-Alive': 'timeout=5',
'Transfer-Encoding': 'chunked',
},
resourceIPAddressSpace: 'Local',
statusCode: 302,
headersText:
'HTTP/1.1 302 Found\r\nlocation: http://localhost:8907/empty.html\r\nDate: Fri, 19 Nov 2021 09:53:58 GMT\r\nConnection: keep-alive\r\nKeep-Alive: timeout=5\r\nTransfer-Encoding: chunked\r\n\r\n',
});
mockCDPSession.emit('Network.requestWillBeSentExtraInfo', {
requestId: '7760711DEFCFA23132D98ABA6B4E175C',
associatedCookies: [],
headers: {
Host: 'localhost:8907',
Connection: 'keep-alive',
'Upgrade-Insecure-Requests': '1',
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/97.0.4691.0 Safari/537.36',
Accept:
'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
'Sec-Fetch-Site': 'none',
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-User': '?1',
'Sec-Fetch-Dest': 'document',
'Accept-Encoding': 'gzip, deflate, br',
},
connectTiming: { requestTime: 2111.561759 },
});
mockCDPSession.emit('Network.responseReceivedExtraInfo', {
requestId: '7760711DEFCFA23132D98ABA6B4E175C',
blockedCookies: [],
headers: {
'Cache-Control': 'no-cache, no-store',
'Content-Type': 'text/html; charset=utf-8',
Date: 'Fri, 19 Nov 2021 09:53:58 GMT',
Connection: 'keep-alive',
'Keep-Alive': 'timeout=5',
'Content-Length': '0',
},
resourceIPAddressSpace: 'Local',
statusCode: 200,
headersText:
'HTTP/1.1 200 OK\r\nCache-Control: no-cache, no-store\r\nContent-Type: text/html; charset=utf-8\r\nDate: Fri, 19 Nov 2021 09:53:58 GMT\r\nConnection: keep-alive\r\nKeep-Alive: timeout=5\r\nContent-Length: 0\r\n\r\n',
});
mockCDPSession.emit('Network.responseReceived', {
requestId: '7760711DEFCFA23132D98ABA6B4E175C',
loaderId: '7760711DEFCFA23132D98ABA6B4E175C',
timestamp: 2111.563565,
type: 'Document',
response: {
url: 'http://localhost:8907/empty.html',
status: 200,
statusText: 'OK',
headers: {
'Cache-Control': 'no-cache, no-store',
'Content-Type': 'text/html; charset=utf-8',
Date: 'Fri, 19 Nov 2021 09:53:58 GMT',
Connection: 'keep-alive',
'Keep-Alive': 'timeout=5',
'Content-Length': '0',
},
mimeType: 'text/html',
connectionReused: true,
connectionId: 322,
remoteIPAddress: '[::1]',
remotePort: 8907,
fromDiskCache: false,
fromServiceWorker: false,
fromPrefetchCache: false,
encodedDataLength: 197,
timing: {
requestTime: 2111.561759,
proxyStart: -1,
proxyEnd: -1,
dnsStart: -1,
dnsEnd: -1,
connectStart: -1,
connectEnd: -1,
sslStart: -1,
sslEnd: -1,
workerStart: -1,
workerReady: -1,
workerFetchStart: -1,
workerRespondWithSettled: -1,
sendStart: 0.148,
sendEnd: 0.19,
pushStart: 0,
pushEnd: 0,
receiveHeadersEnd: 0.925,
},
responseTime: 1.637315638479928e12,
protocol: 'http/1.1',
securityState: 'secure',
},
hasExtraInfo: true,
frameId: '099A5216AF03AAFEC988F214B024DF08',
});
});
it(`should handle "double pause" (crbug.com/1196004) Fetch.requestPaused events for the same Network.requestWillBeSent event`, async () => {
const mockCDPSession = new MockCDPSession();
const manager = new NetworkManager(mockCDPSession, true, {
frame(): Frame | null {
return null;
},
});
manager.setRequestInterception(true);
const requests: HTTPRequest[] = [];
manager.on(NetworkManagerEmittedEvents.Request, (request: HTTPRequest) => {
request.continue();
requests.push(request);
});
/**
* This sequence was taken from an actual CDP session produced by the following
* test script:
*
* const browser = await puppeteer.launch(\{ headless: false \});
* const page = await browser.newPage();
* await page.setCacheEnabled(false);
*
* await page.setRequestInterception(true)
* page.on('request', (interceptedRequest) =\> \{
* interceptedRequest.continue();
* \});
*
* await page.goto('https://www.google.com');
* await browser.close();
*
*/
mockCDPSession.emit('Network.requestWillBeSent', {
requestId: '11ACE9783588040D644B905E8B55285B',
loaderId: '11ACE9783588040D644B905E8B55285B',
documentURL: 'https://www.google.com/',
request: {
url: 'https://www.google.com/',
method: 'GET',
headers: [Object],
mixedContentType: 'none',
initialPriority: 'VeryHigh',
referrerPolicy: 'strict-origin-when-cross-origin',
isSameSite: true,
},
timestamp: 224604.980827,
wallTime: 1637955746.786191,
initiator: { type: 'other' },
redirectHasExtraInfo: false,
type: 'Document',
frameId: '84AC261A351B86932B775B76D1DD79F8',
hasUserGesture: false,
});
mockCDPSession.emit('Fetch.requestPaused', {
requestId: 'interception-job-1.0',
request: {
url: 'https://www.google.com/',
method: 'GET',
headers: [Object],
initialPriority: 'VeryHigh',
referrerPolicy: 'strict-origin-when-cross-origin',
},
frameId: '84AC261A351B86932B775B76D1DD79F8',
resourceType: 'Document',
networkId: '11ACE9783588040D644B905E8B55285B',
});
mockCDPSession.emit('Fetch.requestPaused', {
requestId: 'interception-job-2.0',
request: {
url: 'https://www.google.com/',
method: 'GET',
headers: [Object],
initialPriority: 'VeryHigh',
referrerPolicy: 'strict-origin-when-cross-origin',
},
frameId: '84AC261A351B86932B775B76D1DD79F8',
resourceType: 'Document',
networkId: '11ACE9783588040D644B905E8B55285B',
});
expect(requests.length).toBe(2);
});
it(`should handle Network.responseReceivedExtraInfo event after Network.responseReceived event (github.com/puppeteer/puppeteer/issues/8234)`, async () => {
const mockCDPSession = new MockCDPSession();
const manager = new NetworkManager(mockCDPSession, true, {
frame(): Frame | null {
return null;
},
});
const requests: HTTPRequest[] = [];
manager.on(
NetworkManagerEmittedEvents.RequestFinished,
(request: HTTPRequest) => {
requests.push(request);
}
);
mockCDPSession.emit('Network.requestWillBeSent', {
requestId: '1360.2',
loaderId: '9E86B0282CC98B77FB0ABD49156DDFDD',
documentURL: 'http://this.is.the.start.page.com/',
request: {
url: 'http://this.is.a.test.com:1080/test.js',
method: 'GET',
headers: {
'Accept-Language': 'en-US,en;q=0.9',
Referer: 'http://this.is.the.start.page.com/',
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.0 Safari/537.36',
},
mixedContentType: 'none',
initialPriority: 'High',
referrerPolicy: 'strict-origin-when-cross-origin',
isSameSite: false,
},
timestamp: 10959.020087,
wallTime: 1649712607.861365,
initiator: {
type: 'parser',
url: 'http://this.is.the.start.page.com/',
lineNumber: 9,
columnNumber: 80,
},
redirectHasExtraInfo: false,
type: 'Script',
frameId: '60E6C35E7E519F28E646056820095498',
hasUserGesture: false,
});
mockCDPSession.emit('Network.responseReceived', {
requestId: '1360.2',
loaderId: '9E86B0282CC98B77FB0ABD49156DDFDD',
timestamp: 10959.042529,
type: 'Script',
response: {
url: 'http://this.is.a.test.com:1080',
status: 200,
statusText: 'OK',
headers: {
connection: 'keep-alive',
'content-length': '85862',
},
mimeType: 'text/plain',
connectionReused: false,
connectionId: 119,
remoteIPAddress: '127.0.0.1',
remotePort: 1080,
fromDiskCache: false,
fromServiceWorker: false,
fromPrefetchCache: false,
encodedDataLength: 66,
timing: {
requestTime: 10959.023904,
proxyStart: -1,
proxyEnd: -1,
dnsStart: 0.328,
dnsEnd: 2.183,
connectStart: 2.183,
connectEnd: 2.798,
sslStart: -1,
sslEnd: -1,
workerStart: -1,
workerReady: -1,
workerFetchStart: -1,
workerRespondWithSettled: -1,
sendStart: 2.982,
sendEnd: 3.757,
pushStart: 0,
pushEnd: 0,
receiveHeadersEnd: 16.373,
},
responseTime: 1649712607880.971,
protocol: 'http/1.1',
securityState: 'insecure',
},
hasExtraInfo: true,
frameId: '60E6C35E7E519F28E646056820095498',
});
mockCDPSession.emit('Network.responseReceivedExtraInfo', {
requestId: '1360.2',
blockedCookies: [],
headers: {
connection: 'keep-alive',
'content-length': '85862',
},
resourceIPAddressSpace: 'Private',
statusCode: 200,
headersText:
'HTTP/1.1 200 OK\r\nconnection: keep-alive\r\ncontent-length: 85862\r\n\r\n',
});
mockCDPSession.emit('Network.loadingFinished', {
requestId: '1360.2',
timestamp: 10959.060708,
encodedDataLength: 85928,
shouldReportCorbBlocking: false,
});
expect(requests.length).toBe(1);
});
});