thunderstrike/puppeteer
thunderstrike/puppeteer
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(firefox): basic request interception support (#4034)

Browse Source 
This patch implements `page.setRequestInterception`, `page.continue`
and `page.abort` methods.
This commit is contained in:
Andrey Lushnikov 2019-02-19 14:51:56 -08:00 committed by GitHub
parent 3b180923a6
commit c118b208fa
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 93 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
experimental/puppeteer-firefox/lib/NetworkManager.js
View File

@ -1,4 +1,4 @@
const {helper} = require('./helper'); const {helper, assert, debugError} = require('./helper');
const util = require('util'); const util = require('util');
const EventEmitter = require('events'); const EventEmitter = require('events');
const {Events} = require('./Events'); const {Events} = require('./Events');
@ -15,6 +15,7 @@ class NetworkManager extends EventEmitter {
helper.addEventListener(session, 'Network.requestWillBeSent', this._onRequestWillBeSent.bind(this)), helper.addEventListener(session, 'Network.requestWillBeSent', this._onRequestWillBeSent.bind(this)),
helper.addEventListener(session, 'Network.responseReceived', this._onResponseReceived.bind(this)), helper.addEventListener(session, 'Network.responseReceived', this._onResponseReceived.bind(this)),
helper.addEventListener(session, 'Network.requestFinished', this._onRequestFinished.bind(this)), helper.addEventListener(session, 'Network.requestFinished', this._onRequestFinished.bind(this)),
helper.addEventListener(session, 'Network.requestFailed', this._onRequestFailed.bind(this)),
]; ];
} }
@ -26,6 +27,10 @@ class NetworkManager extends EventEmitter {
this._frameManager = frameManager; this._frameManager = frameManager;
} }
async setRequestInterception(enabled) {
await this._session.send('Network.setRequestInterception', {enabled});
}
_onRequestWillBeSent(event) { _onRequestWillBeSent(event) {
const redirected = event.redirectedFrom ? this._requests.get(event.redirectedFrom) : null; const redirected = event.redirectedFrom ? this._requests.get(event.redirectedFrom) : null;
const frame = redirected ? redirected.frame() : (this._frameManager && event.frameId ? this._frameManager.frame(event.frameId) : null); const frame = redirected ? redirected.frame() : (this._frameManager && event.frameId ? this._frameManager.frame(event.frameId) : null);
@ -37,7 +42,7 @@ class NetworkManager extends EventEmitter {
redirectChain.push(redirected); redirectChain.push(redirected);
this._requests.delete(redirected._id); this._requests.delete(redirected._id);
} }
const request = new Request(frame, redirectChain, event); const request = new Request(this._session, frame, redirectChain, event);
this._requests.set(request._id, request); this._requests.set(request._id, request);
this.emit(Events.NetworkManager.Request, request); this.emit(Events.NetworkManager.Request, request);
} }
@ -61,6 +66,15 @@ class NetworkManager extends EventEmitter {
this._requests.delete(request._id); this._requests.delete(request._id);
this.emit(Events.NetworkManager.RequestFinished, request); this.emit(Events.NetworkManager.RequestFinished, request);
} }
_onRequestFailed(event) {
const request = this._requests.get(event.requestId);
if (!request)
return;
this._requests.delete(request._id);
request._errorText = event.errorCode;
this.emit(Events.NetworkManager.RequestFailed, request);
}
} }
/** /**
@ -94,21 +108,51 @@ const causeToResourceType = {
}; };
class Request { class Request {
constructor(frame, redirectChain, payload) { constructor(session, frame, redirectChain, payload) {
this._session = session;
this._frame = frame; this._frame = frame;
this._id = payload.requestId; this._id = payload.requestId;
this._redirectChain = redirectChain; this._redirectChain = redirectChain;
this._url = payload.url; this._url = payload.url;
this._postData = payload.postData; this._postData = payload.postData;
this._suspended = payload.suspended;
this._response = null; this._response = null;
this._errorText = null;
this._isNavigationRequest = payload.isNavigationRequest; this._isNavigationRequest = payload.isNavigationRequest;
this._method = payload.method; this._method = payload.method;
this._resourceType = causeToResourceType[payload.cause] || 'other'; this._resourceType = causeToResourceType[payload.cause] || 'other';
this._headers = {}; this._headers = {};
this._interceptionHandled = false;
for (const {name, value} of payload.headers) for (const {name, value} of payload.headers)
this._headers[name.toLowerCase()] = value; this._headers[name.toLowerCase()] = value;
} }
failure() {
return this._errorText ? {errorText: this._errorText} : null;
}
async continue() {
assert(this._suspended, 'Request Interception is not enabled!');
assert(!this._interceptionHandled, 'Request is already handled!');
this._interceptionHandled = true;
await this._session.send('Network.resumeSuspendedRequest', {
requestId: this._id,
}).catch(error => {
debugError(error);
});
}
async abort() {
assert(this._suspended, 'Request Interception is not enabled!');
assert(!this._interceptionHandled, 'Request is already handled!');
this._interceptionHandled = true;
await this._session.send('Network.abortSuspendedRequest', {
requestId: this._id,
}).catch(error => {
debugError(error);
});
}
postData() { postData() {
return this._postData; return this._postData;
} }

4
experimental/puppeteer-firefox/lib/Page.js
View File

@ -73,6 +73,10 @@ class Page extends EventEmitter {
}); });
} }
async setRequestInterception(enabled) {
await this._networkManager.setRequestInterception(enabled);
}
/** /**
* @param {(string|Function)} urlOrPredicate * @param {(string|Function)} urlOrPredicate
* @param {!{timeout?: number}=} options * @param {!{timeout?: number}=} options

2
experimental/puppeteer-firefox/package.json
View File

@ -9,7 +9,7 @@
"node": ">=8.9.4" "node": ">=8.9.4"
}, },
"puppeteer": { "puppeteer": {
"firefox_revision": "387ac6bbbe5357d174e9fb3aa9b6f935113c315d" "firefox_revision": "98116977b3f0c936c92e917bdd571c340167a536"
}, },
"scripts": { "scripts": {
"install": "node install.js", "install": "node install.js",

2
test/ignorehttpserrors.spec.js
View File

@ -69,7 +69,7 @@ module.exports.addTests = function({testRunner, expect, defaultBrowserOptions, p
expect(error).toBe(null); expect(error).toBe(null);
expect(response.ok()).toBe(true); expect(response.ok()).toBe(true);
}); });
it_fails_ffox('should work with request interception', async({page, server, httpsServer}) => { it('should work with request interception', async({page, server, httpsServer}) => {
await page.setRequestInterception(true); await page.setRequestInterception(true);
page.on('request', request => request.continue()); page.on('request', request => request.continue());
const response = await page.goto(httpsServer.EMPTY_PAGE); const response = await page.goto(httpsServer.EMPTY_PAGE);

60
test/network.spec.js
View File

@ -243,7 +243,7 @@ module.exports.addTests = function({testRunner, expect, CHROME}) {
await new Promise(x => serverResponse.end('ld!', x)); await new Promise(x => serverResponse.end('ld!', x));
expect(await responseText).toBe('hello world!'); expect(await responseText).toBe('hello world!');
}); });
it_fails_ffox('Page.Events.RequestFailed', async({page, server}) => { it('Page.Events.RequestFailed', async({page, server}) => {
await page.setRequestInterception(true); await page.setRequestInterception(true);
page.on('request', request => { page.on('request', request => {
if (request.url().endsWith('css')) if (request.url().endsWith('css'))
@ -258,7 +258,10 @@ module.exports.addTests = function({testRunner, expect, CHROME}) {
expect(failedRequests[0].url()).toContain('one-style.css'); expect(failedRequests[0].url()).toContain('one-style.css');
expect(failedRequests[0].response()).toBe(null); expect(failedRequests[0].response()).toBe(null);
expect(failedRequests[0].resourceType()).toBe('stylesheet'); expect(failedRequests[0].resourceType()).toBe('stylesheet');
expect(failedRequests[0].failure().errorText).toBe('net::ERR_FAILED'); if (CHROME)
expect(failedRequests[0].failure().errorText).toBe('net::ERR_FAILED');
else
expect(failedRequests[0].failure().errorText).toBe('NS_ERROR_FAILURE');
expect(failedRequests[0].frame()).toBeTruthy(); expect(failedRequests[0].frame()).toBeTruthy();
}); });
it('Page.Events.RequestFinished', async({page, server}) => { it('Page.Events.RequestFinished', async({page, server}) => {
@ -317,7 +320,7 @@ module.exports.addTests = function({testRunner, expect, CHROME}) {
expect(requests.get('script.js').isNavigationRequest()).toBe(false); expect(requests.get('script.js').isNavigationRequest()).toBe(false);
expect(requests.get('style.css').isNavigationRequest()).toBe(false); expect(requests.get('style.css').isNavigationRequest()).toBe(false);
}); });
it_fails_ffox('should work with request interception', async({page, server}) => { it('should work with request interception', async({page, server}) => {
const requests = new Map(); const requests = new Map();
page.on('request', request => { page.on('request', request => {
requests.set(request.url().split('/').pop(), request); requests.set(request.url().split('/').pop(), request);
@ -340,10 +343,14 @@ module.exports.addTests = function({testRunner, expect, CHROME}) {
}); });
}); });
describe_fails_ffox('Page.setRequestInterception', function() { describe('Page.setRequestInterception', function() {
it('should intercept', async({page, server}) => { it('should intercept', async({page, server}) => {
await page.setRequestInterception(true); await page.setRequestInterception(true);
page.on('request', request => { page.on('request', request => {
if (utils.isFavicon(request)) {
request.continue();
return;
}
expect(request.url()).toContain('empty.html'); expect(request.url()).toContain('empty.html');
expect(request.headers()['user-agent']).toBeTruthy(); expect(request.headers()['user-agent']).toBeTruthy();
expect(request.method()).toBe('GET'); expect(request.method()).toBe('GET');
@ -358,7 +365,8 @@ module.exports.addTests = function({testRunner, expect, CHROME}) {
expect(response.ok()).toBe(true); expect(response.ok()).toBe(true);
expect(response.remoteAddress().port).toBe(server.PORT); expect(response.remoteAddress().port).toBe(server.PORT);
}); });
it('should work with intervention headers', async({page, server}) => { // Intervention headers are chrome-specific.
(CHROME ? it : xit)('should work with intervention headers', async({page, server}) => {
server.setRoute('/intervention', (req, res) => res.end(` server.setRoute('/intervention', (req, res) => res.end(`
<script> <script>
document.write('<script src="${server.CROSS_PROCESS_PREFIX}/intervention.js">' + '</scr' + 'ipt>'); document.write('<script src="${server.CROSS_PROCESS_PREFIX}/intervention.js">' + '</scr' + 'ipt>');
@ -395,14 +403,15 @@ module.exports.addTests = function({testRunner, expect, CHROME}) {
await page.setRequestInterception(true); await page.setRequestInterception(true);
const requests = []; const requests = [];
page.on('request', request => { page.on('request', request => {
requests.push(request); if (!utils.isFavicon(request))
requests.push(request);
request.continue(); request.continue();
}); });
await page.goto(server.PREFIX + '/one-style.html'); await page.goto(server.PREFIX + '/one-style.html');
expect(requests[1].url()).toContain('/one-style.css'); expect(requests[1].url()).toContain('/one-style.css');
expect(requests[1].headers().referer).toContain('/one-style.html'); expect(requests[1].headers().referer).toContain('/one-style.html');
}); });
it('should properly return navigation response when URL has cookies', async({page, server}) => { it_fails_ffox('should properly return navigation response when URL has cookies', async({page, server}) => {
// Setup cookie. // Setup cookie.
await page.goto(server.EMPTY_PAGE); await page.goto(server.EMPTY_PAGE);
await page.setCookie({ name: 'foo', value: 'bar'}); await page.setCookie({ name: 'foo', value: 'bar'});
@ -420,7 +429,7 @@ module.exports.addTests = function({testRunner, expect, CHROME}) {
await page.setRequestInterception(false); await page.setRequestInterception(false);
await page.goto(server.EMPTY_PAGE); await page.goto(server.EMPTY_PAGE);
}); });
it('should show custom HTTP headers', async({page, server}) => { it_fails_ffox('should show custom HTTP headers', async({page, server}) => {
await page.setExtraHTTPHeaders({ await page.setExtraHTTPHeaders({
foo: 'bar' foo: 'bar'
}); });
@ -432,7 +441,7 @@ module.exports.addTests = function({testRunner, expect, CHROME}) {
const response = await page.goto(server.EMPTY_PAGE); const response = await page.goto(server.EMPTY_PAGE);
expect(response.ok()).toBe(true); expect(response.ok()).toBe(true);
}); });
it('should works with customizing referer headers', async({page, server}) => { it_fails_ffox('should works with customizing referer headers', async({page, server}) => {
await page.setExtraHTTPHeaders({ 'referer': server.EMPTY_PAGE }); await page.setExtraHTTPHeaders({ 'referer': server.EMPTY_PAGE });
await page.setRequestInterception(true); await page.setRequestInterception(true);
page.on('request', request => { page.on('request', request => {
@ -457,7 +466,7 @@ module.exports.addTests = function({testRunner, expect, CHROME}) {
expect(response.request().failure()).toBe(null); expect(response.request().failure()).toBe(null);
expect(failedRequests).toBe(1); expect(failedRequests).toBe(1);
}); });
it('should be abortable with custom error codes', async({page, server}) => { it_fails_ffox('should be abortable with custom error codes', async({page, server}) => {
await page.setRequestInterception(true); await page.setRequestInterception(true);
page.on('request', request => { page.on('request', request => {
request.abort('internetdisconnected'); request.abort('internetdisconnected');
@ -468,7 +477,7 @@ module.exports.addTests = function({testRunner, expect, CHROME}) {
expect(failedRequest).toBeTruthy(); expect(failedRequest).toBeTruthy();
expect(failedRequest.failure().errorText).toBe('net::ERR_INTERNET_DISCONNECTED'); expect(failedRequest.failure().errorText).toBe('net::ERR_INTERNET_DISCONNECTED');
}); });
it('should send referer', async({page, server}) => { it_fails_ffox('should send referer', async({page, server}) => {
await page.setExtraHTTPHeaders({ await page.setExtraHTTPHeaders({
referer: 'http://google.com/' referer: 'http://google.com/'
}); });
@ -480,7 +489,7 @@ module.exports.addTests = function({testRunner, expect, CHROME}) {
]); ]);
expect(request.headers['referer']).toBe('http://google.com/'); expect(request.headers['referer']).toBe('http://google.com/');
}); });
it('should amend HTTP headers', async({page, server}) => { it_fails_ffox('should amend HTTP headers', async({page, server}) => {
await page.setRequestInterception(true); await page.setRequestInterception(true);
page.on('request', request => { page.on('request', request => {
const headers = Object.assign({}, request.headers()); const headers = Object.assign({}, request.headers());
@ -500,7 +509,10 @@ module.exports.addTests = function({testRunner, expect, CHROME}) {
let error = null; let error = null;
await page.goto(server.EMPTY_PAGE).catch(e => error = e); await page.goto(server.EMPTY_PAGE).catch(e => error = e);
expect(error).toBeTruthy(); expect(error).toBeTruthy();
expect(error.message).toContain('net::ERR_FAILED'); if (CHROME)
expect(error.message).toContain('net::ERR_FAILED');
else
expect(error.message).toContain('NS_ERROR_FAILURE');
}); });
it('should work with redirects', async({page, server}) => { it('should work with redirects', async({page, server}) => {
await page.setRequestInterception(true); await page.setRequestInterception(true);
@ -534,7 +546,8 @@ module.exports.addTests = function({testRunner, expect, CHROME}) {
const requests = []; const requests = [];
page.on('request', request => { page.on('request', request => {
request.continue(); request.continue();
requests.push(request); if (!utils.isFavicon(request))
requests.push(request);
}); });
server.setRedirect('/one-style.css', '/two-style.css'); server.setRedirect('/one-style.css', '/two-style.css');
server.setRedirect('/two-style.css', '/three-style.css'); server.setRedirect('/two-style.css', '/three-style.css');
@ -571,7 +584,10 @@ module.exports.addTests = function({testRunner, expect, CHROME}) {
return e.message; return e.message;
} }
}); });
expect(result).toContain('Failed to fetch'); if (CHROME)
expect(result).toContain('Failed to fetch');
else
expect(result).toContain('NetworkError');
}); });
it('should work with equal requests', async({page, server}) => { it('should work with equal requests', async({page, server}) => {
await page.goto(server.EMPTY_PAGE); await page.goto(server.EMPTY_PAGE);
@ -582,6 +598,10 @@ module.exports.addTests = function({testRunner, expect, CHROME}) {
let spinner = false; let spinner = false;
// Cancel 2nd request. // Cancel 2nd request.
page.on('request', request => { page.on('request', request => {
if (utils.isFavicon(request)) {
request.continue();
return;
}
spinner ? request.abort() : request.continue(); spinner ? request.abort() : request.continue();
spinner = !spinner; spinner = !spinner;
}); });
@ -592,7 +612,7 @@ module.exports.addTests = function({testRunner, expect, CHROME}) {
])); ]));
expect(results).toEqual(['11', 'FAILED', '22']); expect(results).toEqual(['11', 'FAILED', '22']);
}); });
it('should navigate to dataURL and fire dataURL requests', async({page, server}) => { it_fails_ffox('should navigate to dataURL and fire dataURL requests', async({page, server}) => {
await page.setRequestInterception(true); await page.setRequestInterception(true);
const requests = []; const requests = [];
page.on('request', request => { page.on('request', request => {
@ -605,7 +625,7 @@ module.exports.addTests = function({testRunner, expect, CHROME}) {
expect(requests.length).toBe(1); expect(requests.length).toBe(1);
expect(requests[0].url()).toBe(dataURL); expect(requests[0].url()).toBe(dataURL);
}); });
it('should navigate to URL with hash and and fire requests without hash', async({page, server}) => { it_fails_ffox('should navigate to URL with hash and and fire requests without hash', async({page, server}) => {
await page.setRequestInterception(true); await page.setRequestInterception(true);
const requests = []; const requests = [];
page.on('request', request => { page.on('request', request => {
@ -633,7 +653,7 @@ module.exports.addTests = function({testRunner, expect, CHROME}) {
const response = await page.goto(server.PREFIX + '/malformed?rnd=%911'); const response = await page.goto(server.PREFIX + '/malformed?rnd=%911');
expect(response.status()).toBe(200); expect(response.status()).toBe(200);
}); });
it('should work with encoded server - 2', async({page, server}) => { it_fails_ffox('should work with encoded server - 2', async({page, server}) => {
// The requestWillBeSent will report URL as-is, whereas interception will // The requestWillBeSent will report URL as-is, whereas interception will
// report encoded URL for stylesheet. @see crbug.com/759388 // report encoded URL for stylesheet. @see crbug.com/759388
await page.setRequestInterception(true); await page.setRequestInterception(true);
@ -647,7 +667,7 @@ module.exports.addTests = function({testRunner, expect, CHROME}) {
expect(requests.length).toBe(2); expect(requests.length).toBe(2);
expect(requests[1].response().status()).toBe(404); expect(requests[1].response().status()).toBe(404);
}); });
it('should not throw "Invalid Interception Id" if the request was cancelled', async({page, server}) => { it_fails_ffox('should not throw "Invalid Interception Id" if the request was cancelled', async({page, server}) => {
await page.setContent('<iframe></iframe>'); await page.setContent('<iframe></iframe>');
await page.setRequestInterception(true); await page.setRequestInterception(true);
let request = null; let request = null;
@ -673,7 +693,7 @@ module.exports.addTests = function({testRunner, expect, CHROME}) {
await page.goto(server.EMPTY_PAGE); await page.goto(server.EMPTY_PAGE);
expect(error.message).toContain('Request Interception is not enabled'); expect(error.message).toContain('Request Interception is not enabled');
}); });
it('should work with file URLs', async({page, server}) => { it_fails_ffox('should work with file URLs', async({page, server}) => {
await page.setRequestInterception(true); await page.setRequestInterception(true);
const urls = new Set(); const urls = new Set();
page.on('request', request => { page.on('request', request => {