Introduce utils/check_availability.js utility

This patch introduces check_availability.js utility which looks for
available chromium binaries for different revisions and platforms.

This patch also re-factors the chromium downloader scripts so that
it can operate different platforms.

examples/custom-chromium-revision.js

@@ -15,13 +15,13 @@
  */
 
 var Browser = require('../lib/Browser');
-var Downloader = require('../lib/Downloader');
+var Downloader = require('../utils/ChromiumDownloader');
 
 var revision = "464642";
 console.log('Downloading custom chromium revision - ' + revision);
-Downloader.downloadChromium(revision).then(async () => {
+Downloader.downloadRevision(Downloader.currentPlatform(), revision).then(async () => {
     console.log('Done.');
-    var executablePath = Downloader.executablePath(revision);
+    var executablePath = Downloader.revisionInfo(Downloader.currentPlatform(), revision).executablePath;
     var browser1 = new Browser({
         remoteDebuggingPort: 9228,
         executablePath,

install.js

@@ -14,16 +14,16 @@
  * limitations under the License.
  */
 
-var Downloader = require('./lib/Downloader');
+var Downloader = require('./utils/ChromiumDownloader');
 var revision = require('./package').puppeteer.chromium_revision;
 var fs = require('fs');
 var ProgressBar = require('progress');
 
-var executable = Downloader.executablePath(revision);
+// Do nothing if the revision is already downloaded.
-if (fs.existsSync(executable))
+if (Downloader.revisionInfo(Downloader.currentPlatform(), revision))
     return;
 
-Downloader.downloadChromium(revision, onProgress)
+Downloader.downloadRevision(Downloader.currentPlatform(), revision, onProgress)
     .catch(error => {
         console.error('Download failed: ' + error.message);
     });

lib/Browser.js

@@ -20,7 +20,7 @@ var path = require('path');
 var removeRecursive = require('rimraf').sync;
 var Page = require('./Page');
 var childProcess = require('child_process');
-var Downloader = require('./Downloader');
+var Downloader = require('../utils/ChromiumDownloader');
 
 var CHROME_PROFILE_PATH = path.resolve(__dirname, '..', '.dev_profile');
 var browserId = 0;
@@ -55,7 +55,9 @@ class Browser {
             this._chromeExecutable = options.executablePath;
         } else {
             var chromiumRevision = require('../package.json').puppeteer.chromium_revision;
-            this._chromeExecutable = Downloader.executablePath(chromiumRevision);
+            var revisionInfo = Downloader.revisionInfo(Downloader.currentPlatform(), chromiumRevision);
+            console.assert(revisionInfo, 'Chromium revision is not downloaded. Run npm install');
+            this._chromeExecutable = revisionInfo.executablePath;
         }
         if (Array.isArray(options.args))
             this._chromeArguments.push(...options.args);

lib/Downloader.js

@@ -1,123 +0,0 @@
-/**
- * Copyright 2017 Google Inc. All rights reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-var os = require('os');
-var https = require('https');
-var fs = require('fs');
-var path = require('path');
-var extract = require('extract-zip');
-var util = require('util');
-
-var CHROMIUM_PATH = path.join(__dirname, '..', '.local-chromium');
-
-var downloadURLs = {
-    linux: 'https://storage.googleapis.com/chromium-browser-snapshots/Linux_x64/%d/chrome-linux.zip',
-    darwin: 'https://storage.googleapis.com/chromium-browser-snapshots/Mac/%d/chrome-mac.zip',
-    win32: 'https://storage.googleapis.com/chromium-browser-snapshots/Win/%d/chrome-win32.zip',
-    win64: 'https://storage.googleapis.com/chromium-browser-snapshots/Win_x64/%d/chrome-win32.zip',
-};
-
-module.exports = {
-    downloadChromium,
-    executablePath,
-};
-
-/**
- * @param {string} revision
- * @param {?function(number, number)} progressCallback
- * @return {!Promise}
- */
-async function downloadChromium(revision, progressCallback) {
-    var url = null;
-    var platform = os.platform();
-    if (platform === 'darwin')
-        url = downloadURLs.darwin;
-    else if (platform === 'linux')
-        url = downloadURLs.linux;
-    else if (platform === 'win32')
-        url = os.arch() === 'x64' ? downloadURLs.win64 : downloadURLs.win32;
-    console.assert(url, `Unsupported platform: ${platform}`);
-    url = util.format(url, revision);
-    var zipPath = path.join(CHROMIUM_PATH, `download-${revision}.zip`);
-    var folderPath = path.join(CHROMIUM_PATH, revision);
-    if (fs.existsSync(folderPath))
-        return;
-    try {
-        if (!fs.existsSync(CHROMIUM_PATH))
-            fs.mkdirSync(CHROMIUM_PATH);
-        await downloadFile(url, zipPath, progressCallback);
-        await extractZip(zipPath, folderPath);
-    } finally {
-        if (fs.existsSync(zipPath))
-            fs.unlinkSync(zipPath);
-    }
-}
-
-/**
- * @return {string}
- */
-function executablePath(revision) {
-    var platform = os.platform();
-    if (platform === 'darwin')
-        return path.join(CHROMIUM_PATH, revision, 'chrome-mac', 'Chromium.app', 'Contents', 'MacOS', 'Chromium');
-    if (platform === 'linux')
-        return path.join(CHROMIUM_PATH, revision, 'chrome-linux', 'chrome');
-    if (platform === 'win32')
-        return path.join(CHROMIUM_PATH, revision, 'chrome-win32', 'chrome.exe');
-    throw new Error(`Unsupported platform: ${platform}`);
-}
-
-/**
- * @param {string} url
- * @param {string} destinationPath
- * @param {?function(number, number)} progressCallback
- * @return {!Promise}
- */
-function downloadFile(url, destinationPath, progressCallback) {
-    var fulfill, reject;
-    var promise = new Promise((x, y) => { fulfill = x; reject = y; });
-    var request = https.get(url, response => {
-        if (response.statusCode !== 200) {
-            var error = new Error(`Download failed: server returned code ${response.statusCode}. URL: ${url}`);
-            // consume response data to free up memory
-            response.resume();
-            reject(error);
-            return;
-        }
-        var file = fs.createWriteStream(destinationPath);
-        file.on('finish', () => fulfill());
-        file.on('error', error => reject(error));
-        response.pipe(file);
-        var totalBytes = parseInt(response.headers['content-length'], 10);
-        if (progressCallback)
-            response.on('data', onData.bind(null, totalBytes));
-    });
-    request.on('error', error => reject(error));
-    return promise;
-
-    function onData(totalBytes, chunk) {
-        progressCallback(totalBytes, chunk.length);
-    }
-}
-
-/**
- * @param {string} zipPath
- * @param {string} folderPath
- * @return {!Promise<?Error>}
- */
-function extractZip(zipPath, folderPath) {
-    return new Promise(fulfill => extract(zipPath, {dir: folderPath}, fulfill));
-}

utils/ChromiumDownloader.js

@@ -0,0 +1,180 @@
+/**
+ * Copyright 2017 Google Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+var os = require('os');
+var https = require('https');
+var fs = require('fs');
+var path = require('path');
+var extract = require('extract-zip');
+var util = require('util');
+var URL = require('url');
+
+var CHROMIUM_PATH = path.join(__dirname, '..', '.local-chromium');
+
+var downloadURLs = {
+    linux: 'https://storage.googleapis.com/chromium-browser-snapshots/Linux_x64/%d/chrome-linux.zip',
+    mac: 'https://storage.googleapis.com/chromium-browser-snapshots/Mac/%d/chrome-mac.zip',
+    win32: 'https://storage.googleapis.com/chromium-browser-snapshots/Win/%d/chrome-win32.zip',
+    win64: 'https://storage.googleapis.com/chromium-browser-snapshots/Win_x64/%d/chrome-win32.zip',
+};
+
+module.exports = {
+    /**
+     * @return {!Array<string>}
+     */
+    supportedPlatforms: function() {
+        return Object.keys(downloadURLs);
+    },
+
+    /**
+     * @return {string}
+     */
+    currentPlatform: function() {
+        var platform = os.platform();
+        if (platform === 'darwin')
+            return 'mac';
+        if (platform === 'linux')
+            return 'linux';
+        if (platform === 'win32')
+            return os.arch() === 'x64' ? 'win64' : 'win32';
+        return '';
+    },
+
+    /**
+     * @param {string} platform
+     * @param {string} revision
+     * @return {!Promise<boolean>}
+     */
+    canDownloadRevision: function(platform, revision) {
+        console.assert(downloadURLs[platform], 'Unknown platform: ' + platform);
+        var url = URL.parse(util.format(downloadURLs[platform], revision));
+        var options = {
+            method: 'HEAD',
+            host: url.host,
+            path: url.pathname,
+        };
+        var resolve;
+        var promise = new Promise(x => resolve = x);
+        var request = https.request(options, response => {
+            resolve(response.statusCode === 200);
+        });
+        request.on('error', error => {
+            console.error(error);
+            resolve(false);
+        });
+        request.end();
+        return promise;
+    },
+
+    /**
+     * @param {string} platform
+     * @param {string} revision
+     * @param {?function(number, number)} progressCallback
+     * @return {!Promise}
+     */
+    downloadRevision: async function(platform, revision, progressCallback) {
+        var url = downloadURLs[platform];
+        console.assert(url, `Unsupported platform: ${platform}`);
+        url = util.format(url, revision);
+        var zipPath = path.join(CHROMIUM_PATH, `download-${platform}-${revision}.zip`);
+        var folderPath = getFolderPath(platform, revision);
+        if (fs.existsSync(folderPath))
+            return;
+        try {
+            if (!fs.existsSync(CHROMIUM_PATH))
+                fs.mkdirSync(CHROMIUM_PATH);
+            await downloadFile(url, zipPath, progressCallback);
+            await extractZip(zipPath, folderPath);
+        } finally {
+            if (fs.existsSync(zipPath))
+                fs.unlinkSync(zipPath);
+        }
+    },
+
+    /**
+     * @param {string} platform
+     * @param {string} revision
+     * @return {?{executablePath: string}}
+     */
+    revisionInfo: function(platform, revision) {
+        var folderPath = getFolderPath(platform, revision);
+        if (!fs.existsSync(folderPath))
+            return null;
+        var executablePath = '';
+        if (platform === 'mac')
+            executablePath = path.join(folderPath, 'chrome-mac', 'Chromium.app', 'Contents', 'MacOS', 'Chromium');
+        else if (platform === 'linux')
+            executablePath = path.join(folderPath, 'chrome-linux', 'chrome');
+        else if (platform === 'win32' || platform === 'win64')
+            executablePath = path.join(folderPath, 'chrome-win32', 'chrome.exe');
+        else
+            throw 'Unsupported platfrom: ' + platfrom;
+        return {
+            executablePath: executablePath
+        };
+    },
+};
+
+/**
+ * @param {string} platform
+ * @param {number} revision
+ * @return {string}
+ */
+function getFolderPath(platform, revision) {
+    return path.join(CHROMIUM_PATH, platform + '-' + revision);
+}
+
+/**
+ * @param {string} url
+ * @param {string} destinationPath
+ * @param {?function(number, number)} progressCallback
+ * @return {!Promise}
+ */
+function downloadFile(url, destinationPath, progressCallback) {
+    var fulfill, reject;
+    var promise = new Promise((x, y) => { fulfill = x; reject = y; });
+    var request = https.get(url, response => {
+        if (response.statusCode !== 200) {
+            var error = new Error(`Download failed: server returned code ${response.statusCode}. URL: ${url}`);
+            // consume response data to free up memory
+            response.resume();
+            reject(error);
+            return;
+        }
+        var file = fs.createWriteStream(destinationPath);
+        file.on('finish', () => fulfill());
+        file.on('error', error => reject(error));
+        response.pipe(file);
+        var totalBytes = parseInt(response.headers['content-length'], 10);
+        if (progressCallback)
+            response.on('data', onData.bind(null, totalBytes));
+    });
+    request.on('error', error => reject(error));
+    return promise;
+
+    function onData(totalBytes, chunk) {
+        progressCallback(totalBytes, chunk.length);
+    }
+}
+
+/**
+ * @param {string} zipPath
+ * @param {string} folderPath
+ * @return {!Promise<?Error>}
+ */
+function extractZip(zipPath, folderPath) {
+    return new Promise(fulfill => extract(zipPath, {dir: folderPath}, fulfill));
+}

utils/check_availability.js

@@ -0,0 +1,196 @@
+#!/usr/bin/env node
+/**
+ * Copyright 2017 Google Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+var Downloader = require('./ChromiumDownloader');
+var https = require('https');
+var OMAHA_PROXY = 'https://omahaproxy.appspot.com/all.json';
+
+var colors = {
+    reset: "\x1b[0m",
+    red: "\x1b[31m",
+    green: "\x1b[32m",
+    yellow: "\x1b[33m"
+};
+
+class Table {
+    /**
+     * @param {!Array<number>} columnWidths
+     */
+    constructor(columnWidths) {
+        this.widths = columnWidths;
+    }
+
+    /**
+     * @param {!Array<string>} values
+     */
+    drawRow(values) {
+        console.assert(values.length === this.widths.length);
+        var row = '';
+        for (var i = 0; i < values.length; ++i)
+            row += padCenter(values[i], this.widths[i]);
+        console.log(row);
+    }
+}
+
+if (process.argv.length === 2) {
+    checkOmahaProxyAvailability();
+    return;
+}
+if (process.argv.length !== 4) {
+    console.log(`
+  Usage: node check_revisions.js [fromRevision] [toRevision]
+
+This script checks availability of different prebuild chromium revisions.
+Running command without arguments will check against omahaproxy revisions.`);
+    return;
+}
+
+var fromRevision = parseInt(process.argv[2], 10);
+var toRevision = parseInt(process.argv[3], 10);
+checkRangeAvailability(fromRevision, toRevision);
+
+/**
+ * @return {!Promise}
+ */
+async function checkOmahaProxyAvailability() {
+    console.log('Fetching revisions from ' + OMAHA_PROXY);
+    var platforms = await loadJSON(OMAHA_PROXY);
+    if (!platforms) {
+        console.error('ERROR: failed to fetch chromium revisions from omahaproxy.');
+        return;
+    }
+    var table = new Table([27, 7, 7, 7, 7]);
+    table.drawRow([''].concat(Downloader.supportedPlatforms()));
+    for (var platform of platforms) {
+        // Trust only to the main platforms.
+        if (platform.os !== 'mac' && platform.os !== 'win' && platform.os !== 'win64' && platform.os !== 'linux')
+            continue;
+        var osName = platform.os === 'win' ? 'win32' : platform.os;
+        for (var version of platform.versions) {
+            if (version.channel !== 'dev' && version.channel !== 'beta' && version.channel !== 'canary' && version.channel !== 'stable')
+                continue;
+            var revisionName = padLeft('[' + osName + ' ' + version.channel + ']', 15);
+            var revision = parseInt(version.branch_base_position, 10);
+            await checkAndDrawRevisionAvailability(table, revisionName, revision);
+        }
+    }
+}
+
+/**
+ * @param {number} fromRevision
+ * @param {number} toRevision
+ * @return {!Promise}
+ */
+async function checkRangeAvailability(fromRevision, toRevision) {
+    var table = new Table([10, 7, 7, 7, 7]);
+    table.drawRow([''].concat(Downloader.supportedPlatforms()));
+    var inc = fromRevision < toRevision ? 1 : -1;
+    for (var revision = fromRevision; revision !== toRevision; revision += inc)
+        await checkAndDrawRevisionAvailability(table, '', revision);
+}
+
+/**
+ * @param {!Table} table
+ * @param {string} name
+ * @param {number} revision
+ * @return {!Promise}
+ */
+async function checkAndDrawRevisionAvailability(table, name, revision) {
+    var promises = [];
+    for (var platform of Downloader.supportedPlatforms())
+        promises.push(Downloader.canDownloadRevision(platform, revision));
+    var availability = await Promise.all(promises);
+    var allAvailable = availability.every(e => !!e);
+    var values = [name + ' ' + (allAvailable ? colors.green + revision + colors.reset : revision)];
+    for (var i = 0; i < availability.length; ++i) {
+        var decoration = availability[i] ? '+' : '-';
+        var color = availability[i] ? colors.green : colors.red;
+        values.push(color + decoration + colors.reset);
+    }
+    table.drawRow(values);
+}
+
+/**
+ * @param {string} url
+ * @return {!Promise<?Object>}
+ */
+function loadJSON(url) {
+    var resolve;
+    var promise = new Promise(x => resolve = x);
+    https.get(url, response => {
+        if (response.statusCode !== 200) {
+            resolve(null);
+            return;
+        }
+        var body = '';
+        response.on('data', function(chunk){
+            body += chunk;
+        });
+        response.on('end', function(){
+            var json = JSON.parse(body);
+            resolve(json);
+        });
+    }).on('error', function(e){
+          console.error('Error fetching json: ' + e);
+          resolve(null);
+    });
+    return promise;
+}
+
+/**
+ * @param {number} size
+ * @return {string}
+ */
+function spaceString(size) {
+    return new Array(size).fill(' ').join('');
+}
+
+/**
+ * @param {string} text
+ * @return {string}
+ */
+function filterOutColors(text) {
+    for (var colorName in colors) {
+        var color = colors[colorName];
+        text = text.replace(color, '');
+    }
+    return text;
+}
+
+/**
+ * @param {string} text
+ * @param {number} length
+ * @return {string}
+ */
+function padLeft(text, length) {
+    var printableCharacters = filterOutColors(text);
+    return printableCharacters.length >= length ? text : spaceString(length - text.length) + text;
+}
+
+/**
+ * @param {string} text
+ * @param {number} length
+ * @return {string}
+ */
+function padCenter(text, length) {
+    var printableCharacters = filterOutColors(text);
+    if (printableCharacters.length >= length)
+        return text;
+    var left = Math.floor((length - printableCharacters.length) / 2);
+    var right = Math.ceil((length - printableCharacters.length) / 2);
+    return spaceString(left) + text + spaceString(right);
+}