thunderstrike/puppeteer
thunderstrike/puppeteer
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Introduce utils/check_availability.js utility

Browse Source 
This patch introduces check_availability.js utility which looks for
available chromium binaries for different revisions and platforms.

This patch also re-factors the chromium downloader scripts so that
it can operate different platforms.
This commit is contained in:
Andrey Lushnikov 2017-05-14 20:05:41 -07:00
parent 6c0e3ebe3e
commit 2b50d8cc32
6 changed files with 387 additions and 132 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
examples/custom-chromium-revision.js
View File

@ -15,13 +15,13 @@
*/
var Browser = require('../lib/Browser');
var Downloader = require('../lib/Downloader');
var Downloader = require('../utils/ChromiumDownloader');
var revision = "464642";
console.log('Downloading custom chromium revision - ' + revision);
Downloader.downloadChromium(revision).then(async () => {
Downloader.downloadRevision(Downloader.currentPlatform(), revision).then(async () => {
console.log('Done.');
var executablePath = Downloader.executablePath(revision);
var executablePath = Downloader.revisionInfo(Downloader.currentPlatform(), revision).executablePath;
var browser1 = new Browser({
remoteDebuggingPort: 9228,
executablePath,

8
install.js
View File

@ -14,16 +14,16 @@
* limitations under the License.
*/
var Downloader = require('./lib/Downloader');
var Downloader = require('./utils/ChromiumDownloader');
var revision = require('./package').puppeteer.chromium_revision;
var fs = require('fs');
var ProgressBar = require('progress');
var executable = Downloader.executablePath(revision);
if (fs.existsSync(executable))
// Do nothing if the revision is already downloaded.
if (Downloader.revisionInfo(Downloader.currentPlatform(), revision))
return;
Downloader.downloadChromium(revision, onProgress)
Downloader.downloadRevision(Downloader.currentPlatform(), revision, onProgress)
.catch(error => {
console.error('Download failed: ' + error.message);
});

6
lib/Browser.js
View File

@ -20,7 +20,7 @@ var path = require('path');
var removeRecursive = require('rimraf').sync;
var Page = require('./Page');
var childProcess = require('child_process');
var Downloader = require('./Downloader');
var Downloader = require('../utils/ChromiumDownloader');
var CHROME_PROFILE_PATH = path.resolve(__dirname, '..', '.dev_profile');
var browserId = 0;
@ -55,7 +55,9 @@ class Browser {
this._chromeExecutable = options.executablePath;
} else {
var chromiumRevision = require('../package.json').puppeteer.chromium_revision;
this._chromeExecutable = Downloader.executablePath(chromiumRevision);
var revisionInfo = Downloader.revisionInfo(Downloader.currentPlatform(), chromiumRevision);
console.assert(revisionInfo, 'Chromium revision is not downloaded. Run npm install');
this._chromeExecutable = revisionInfo.executablePath;
}
if (Array.isArray(options.args))
this._chromeArguments.push(...options.args);

123
lib/Downloader.js
View File

@ -1,123 +0,0 @@
/**
* Copyright 2017 Google Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
var os = require('os');
var https = require('https');
var fs = require('fs');
var path = require('path');
var extract = require('extract-zip');
var util = require('util');
var CHROMIUM_PATH = path.join(__dirname, '..', '.local-chromium');
var downloadURLs = {
linux: 'https://storage.googleapis.com/chromium-browser-snapshots/Linux_x64/%d/chrome-linux.zip',
darwin: 'https://storage.googleapis.com/chromium-browser-snapshots/Mac/%d/chrome-mac.zip',
win32: 'https://storage.googleapis.com/chromium-browser-snapshots/Win/%d/chrome-win32.zip',
win64: 'https://storage.googleapis.com/chromium-browser-snapshots/Win_x64/%d/chrome-win32.zip',
};
module.exports = {
downloadChromium,
executablePath,
};
/**
* @param {string} revision
* @param {?function(number, number)} progressCallback
* @return {!Promise}
*/
async function downloadChromium(revision, progressCallback) {
var url = null;
var platform = os.platform();
if (platform === 'darwin')
url = downloadURLs.darwin;
else if (platform === 'linux')
url = downloadURLs.linux;
else if (platform === 'win32')
url = os.arch() === 'x64' ? downloadURLs.win64 : downloadURLs.win32;
console.assert(url, `Unsupported platform: ${platform}`);
url = util.format(url, revision);
var zipPath = path.join(CHROMIUM_PATH, `download-${revision}.zip`);
var folderPath = path.join(CHROMIUM_PATH, revision);
if (fs.existsSync(folderPath))
return;
try {
if (!fs.existsSync(CHROMIUM_PATH))
fs.mkdirSync(CHROMIUM_PATH);
await downloadFile(url, zipPath, progressCallback);
await extractZip(zipPath, folderPath);
} finally {
if (fs.existsSync(zipPath))
fs.unlinkSync(zipPath);
}
}
/**
* @return {string}
*/
function executablePath(revision) {
var platform = os.platform();
if (platform === 'darwin')
return path.join(CHROMIUM_PATH, revision, 'chrome-mac', 'Chromium.app', 'Contents', 'MacOS', 'Chromium');
if (platform === 'linux')
return path.join(CHROMIUM_PATH, revision, 'chrome-linux', 'chrome');
if (platform === 'win32')
return path.join(CHROMIUM_PATH, revision, 'chrome-win32', 'chrome.exe');
throw new Error(`Unsupported platform: ${platform}`);
}
/**
* @param {string} url
* @param {string} destinationPath
* @param {?function(number, number)} progressCallback
* @return {!Promise}
*/
function downloadFile(url, destinationPath, progressCallback) {
var fulfill, reject;
var promise = new Promise((x, y) => { fulfill = x; reject = y; });
var request = https.get(url, response => {
if (response.statusCode !== 200) {
var error = new Error(`Download failed: server returned code ${response.statusCode}. URL: ${url}`);
// consume response data to free up memory
response.resume();
reject(error);
return;
}
var file = fs.createWriteStream(destinationPath);
file.on('finish', () => fulfill());
file.on('error', error => reject(error));
response.pipe(file);
var totalBytes = parseInt(response.headers['content-length'], 10);
if (progressCallback)
response.on('data', onData.bind(null, totalBytes));
});
request.on('error', error => reject(error));
return promise;
function onData(totalBytes, chunk) {
progressCallback(totalBytes, chunk.length);
}
}
/**
* @param {string} zipPath
* @param {string} folderPath
* @return {!Promise<?Error>}
*/
function extractZip(zipPath, folderPath) {
return new Promise(fulfill => extract(zipPath, {dir: folderPath}, fulfill));
}

180
utils/ChromiumDownloader.js Normal file
View File

@ -0,0 +1,180 @@
/**
* Copyright 2017 Google Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
var os = require('os');
var https = require('https');
var fs = require('fs');
var path = require('path');
var extract = require('extract-zip');
var util = require('util');
var URL = require('url');
var CHROMIUM_PATH = path.join(__dirname, '..', '.local-chromium');
var downloadURLs = {
linux: 'https://storage.googleapis.com/chromium-browser-snapshots/Linux_x64/%d/chrome-linux.zip',
mac: 'https://storage.googleapis.com/chromium-browser-snapshots/Mac/%d/chrome-mac.zip',
win32: 'https://storage.googleapis.com/chromium-browser-snapshots/Win/%d/chrome-win32.zip',
win64: 'https://storage.googleapis.com/chromium-browser-snapshots/Win_x64/%d/chrome-win32.zip',
};
module.exports = {
/**
* @return {!Array<string>}
*/
supportedPlatforms: function() {
return Object.keys(downloadURLs);
},
/**
* @return {string}
*/
currentPlatform: function() {
var platform = os.platform();
if (platform === 'darwin')
return 'mac';
if (platform === 'linux')
return 'linux';
if (platform === 'win32')
return os.arch() === 'x64' ? 'win64' : 'win32';
return '';
},
/**
* @param {string} platform
* @param {string} revision
* @return {!Promise<boolean>}
*/
canDownloadRevision: function(platform, revision) {
console.assert(downloadURLs[platform], 'Unknown platform: ' + platform);
var url = URL.parse(util.format(downloadURLs[platform], revision));
var options = {
method: 'HEAD',
host: url.host,
path: url.pathname,
};
var resolve;
var promise = new Promise(x => resolve = x);
var request = https.request(options, response => {
resolve(response.statusCode === 200);
});
request.on('error', error => {
console.error(error);
resolve(false);
});
request.end();
return promise;
},
/**
* @param {string} platform
* @param {string} revision
* @param {?function(number, number)} progressCallback
* @return {!Promise}
*/
downloadRevision: async function(platform, revision, progressCallback) {
var url = downloadURLs[platform];
console.assert(url, `Unsupported platform: ${platform}`);
url = util.format(url, revision);
var zipPath = path.join(CHROMIUM_PATH, `download-${platform}-${revision}.zip`);
var folderPath = getFolderPath(platform, revision);
if (fs.existsSync(folderPath))
return;
try {
if (!fs.existsSync(CHROMIUM_PATH))
fs.mkdirSync(CHROMIUM_PATH);
await downloadFile(url, zipPath, progressCallback);
await extractZip(zipPath, folderPath);
} finally {
if (fs.existsSync(zipPath))
fs.unlinkSync(zipPath);
}
},
/**
* @param {string} platform
* @param {string} revision
* @return {?{executablePath: string}}
*/
revisionInfo: function(platform, revision) {
var folderPath = getFolderPath(platform, revision);
if (!fs.existsSync(folderPath))
return null;
var executablePath = '';
if (platform === 'mac')
executablePath = path.join(folderPath, 'chrome-mac', 'Chromium.app', 'Contents', 'MacOS', 'Chromium');
else if (platform === 'linux')
executablePath = path.join(folderPath, 'chrome-linux', 'chrome');
else if (platform === 'win32' || platform === 'win64')
executablePath = path.join(folderPath, 'chrome-win32', 'chrome.exe');
else
throw 'Unsupported platfrom: ' + platfrom;
return {
executablePath: executablePath
};
},
};
/**
* @param {string} platform
* @param {number} revision
* @return {string}
*/
function getFolderPath(platform, revision) {
return path.join(CHROMIUM_PATH, platform + '-' + revision);
}
/**
* @param {string} url
* @param {string} destinationPath
* @param {?function(number, number)} progressCallback
* @return {!Promise}
*/
function downloadFile(url, destinationPath, progressCallback) {
var fulfill, reject;
var promise = new Promise((x, y) => { fulfill = x; reject = y; });
var request = https.get(url, response => {
if (response.statusCode !== 200) {
var error = new Error(`Download failed: server returned code ${response.statusCode}. URL: ${url}`);
// consume response data to free up memory
response.resume();
reject(error);
return;
}
var file = fs.createWriteStream(destinationPath);
file.on('finish', () => fulfill());
file.on('error', error => reject(error));
response.pipe(file);
var totalBytes = parseInt(response.headers['content-length'], 10);
if (progressCallback)
response.on('data', onData.bind(null, totalBytes));
});
request.on('error', error => reject(error));
return promise;
function onData(totalBytes, chunk) {
progressCallback(totalBytes, chunk.length);
}
}
/**
* @param {string} zipPath
* @param {string} folderPath
* @return {!Promise<?Error>}
*/
function extractZip(zipPath, folderPath) {
return new Promise(fulfill => extract(zipPath, {dir: folderPath}, fulfill));
}

196
utils/check_availability.js Executable file
View File

@ -0,0 +1,196 @@
#!/usr/bin/env node
/**
* Copyright 2017 Google Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
var Downloader = require('./ChromiumDownloader');
var https = require('https');
var OMAHA_PROXY = 'https://omahaproxy.appspot.com/all.json';
var colors = {
reset: "\x1b[0m",
red: "\x1b[31m",
green: "\x1b[32m",
yellow: "\x1b[33m"
};
class Table {
/**
* @param {!Array<number>} columnWidths
*/
constructor(columnWidths) {
this.widths = columnWidths;
}
/**
* @param {!Array<string>} values
*/
drawRow(values) {
console.assert(values.length === this.widths.length);
var row = '';
for (var i = 0; i < values.length; ++i)
row += padCenter(values[i], this.widths[i]);
console.log(row);
}
}
if (process.argv.length === 2) {
checkOmahaProxyAvailability();
return;
}
if (process.argv.length !== 4) {
console.log(`
Usage: node check_revisions.js [fromRevision] [toRevision]
This script checks availability of different prebuild chromium revisions.
Running command without arguments will check against omahaproxy revisions.`);
return;
}
var fromRevision = parseInt(process.argv[2], 10);
var toRevision = parseInt(process.argv[3], 10);
checkRangeAvailability(fromRevision, toRevision);
/**
* @return {!Promise}
*/
async function checkOmahaProxyAvailability() {
console.log('Fetching revisions from ' + OMAHA_PROXY);
var platforms = await loadJSON(OMAHA_PROXY);
if (!platforms) {
console.error('ERROR: failed to fetch chromium revisions from omahaproxy.');
return;
}
var table = new Table([27, 7, 7, 7, 7]);
table.drawRow([''].concat(Downloader.supportedPlatforms()));
for (var platform of platforms) {
// Trust only to the main platforms.
if (platform.os !== 'mac' && platform.os !== 'win' && platform.os !== 'win64' && platform.os !== 'linux')
continue;
var osName = platform.os === 'win' ? 'win32' : platform.os;
for (var version of platform.versions) {
if (version.channel !== 'dev' && version.channel !== 'beta' && version.channel !== 'canary' && version.channel !== 'stable')
continue;
var revisionName = padLeft('[' + osName + ' ' + version.channel + ']', 15);
var revision = parseInt(version.branch_base_position, 10);
await checkAndDrawRevisionAvailability(table, revisionName, revision);
}
}
}
/**
* @param {number} fromRevision
* @param {number} toRevision
* @return {!Promise}
*/
async function checkRangeAvailability(fromRevision, toRevision) {
var table = new Table([10, 7, 7, 7, 7]);
table.drawRow([''].concat(Downloader.supportedPlatforms()));
var inc = fromRevision < toRevision ? 1 : -1;
for (var revision = fromRevision; revision !== toRevision; revision += inc)
await checkAndDrawRevisionAvailability(table, '', revision);
}
/**
* @param {!Table} table
* @param {string} name
* @param {number} revision
* @return {!Promise}
*/
async function checkAndDrawRevisionAvailability(table, name, revision) {
var promises = [];
for (var platform of Downloader.supportedPlatforms())
promises.push(Downloader.canDownloadRevision(platform, revision));
var availability = await Promise.all(promises);
var allAvailable = availability.every(e => !!e);
var values = [name + ' ' + (allAvailable ? colors.green + revision + colors.reset : revision)];
for (var i = 0; i < availability.length; ++i) {
var decoration = availability[i] ? '+' : '-';
var color = availability[i] ? colors.green : colors.red;
values.push(color + decoration + colors.reset);
}
table.drawRow(values);
}
/**
* @param {string} url
* @return {!Promise<?Object>}
*/
function loadJSON(url) {
var resolve;
var promise = new Promise(x => resolve = x);
https.get(url, response => {
if (response.statusCode !== 200) {
resolve(null);
return;
}
var body = '';
response.on('data', function(chunk){
body += chunk;
});
response.on('end', function(){
var json = JSON.parse(body);
resolve(json);
});
}).on('error', function(e){
console.error('Error fetching json: ' + e);
resolve(null);
});
return promise;
}
/**
* @param {number} size
* @return {string}
*/
function spaceString(size) {
return new Array(size).fill(' ').join('');
}
/**
* @param {string} text
* @return {string}
*/
function filterOutColors(text) {
for (var colorName in colors) {
var color = colors[colorName];
text = text.replace(color, '');
}
return text;
}
/**
* @param {string} text
* @param {number} length
* @return {string}
*/
function padLeft(text, length) {
var printableCharacters = filterOutColors(text);
return printableCharacters.length >= length ? text : spaceString(length - text.length) + text;
}
/**
* @param {string} text
* @param {number} length
* @return {string}
*/
function padCenter(text, length) {
var printableCharacters = filterOutColors(text);
if (printableCharacters.length >= length)
return text;
var left = Math.floor((length - printableCharacters.length) / 2);
var right = Math.ceil((length - printableCharacters.length) / 2);
return spaceString(left) + text + spaceString(right);
}