refactor: use @puppeteer/browsers for install (#9898)

Co-authored-by: Nikolay Vitkov <34244704+Lightning00Blade@users.noreply.github.com>

.github/workflows/ci.yml

@@ -143,7 +143,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: ~/.cache/puppeteer/chrome
-          key: ${{ runner.os }}-chromium-${{ hashFiles('packages/puppeteer-core/src/revisions.ts') }}
+          key: ${{ runner.os }}-chromium-${{ hashFiles('packages/puppeteer-core/src/revisions.ts') }}-${{ hashFiles('packages/puppeteer/src/node/install.ts') }}
       - name: Install Chromium
         run: npm run postinstall
       - name: Tests types
@@ -211,7 +211,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: ~/.cache/puppeteer/firefox
-          key: ${{ runner.os }}-firefox-${{ hashFiles('packages/puppeteer-core/src/revisions.ts') }}
+          key: ${{ runner.os }}-firefox-${{ hashFiles('packages/puppeteer-core/src/revisions.ts') }}-${{ hashFiles('packages/puppeteer/src/node/install.ts') }}
       - name: Install Firefox
         env:
           PUPPETEER_PRODUCT: firefox

docker/Dockerfile

@@ -17,11 +17,11 @@ USER pptruser
 
 WORKDIR /home/pptruser
 
-COPY puppeteer-latest.tgz puppeteer-core-latest.tgz ./
+COPY puppeteer-browsers-latest.tgz puppeteer-latest.tgz puppeteer-core-latest.tgz ./
 
-# Install puppeteer and puppeteer-core into /home/pptruser/node_modules.
+# Install @puppeteer/browsers, puppeteer and puppeteer-core into /home/pptruser/node_modules.
-RUN npm i ./puppeteer-core-latest.tgz ./puppeteer-latest.tgz \
+RUN npm i ./puppeteer-browsers-latest.tgz ./puppeteer-core-latest.tgz ./puppeteer-latest.tgz \
-    && rm ./puppeteer-core-latest.tgz ./puppeteer-latest.tgz \
+    && rm ./puppeteer-browsers-latest.tgz ./puppeteer-core-latest.tgz ./puppeteer-latest.tgz \
     && (node -e "require('child_process').execSync(require('puppeteer').executablePath() + ' --credits', {stdio: 'inherit'})" > THIRD_PARTY_NOTICES)
 
 CMD ["google-chrome-stable"]

docker/pack.sh

@@ -7,10 +7,12 @@ set -e
 
 cd docker
 
-npm pack --workspace puppeteer --workspace puppeteer-core --pack-destination .
+npm pack --workspace puppeteer --workspace puppeteer-core --workspace @puppeteer/browsers --pack-destination .
 
 rm -f puppeteer-core-latest.tgz
 rm -f puppeteer-latest.tgz
+rm -f puppeteer-browsers-latest.tgz
 
 mv puppeteer-core-*.tgz puppeteer-core-latest.tgz
+mv puppeteer-browsers-*.tgz puppeteer-browsers-latest.tgz
 mv puppeteer-[0-9]*.tgz puppeteer-latest.tgz

package-lock.json

@@ -9317,7 +9317,7 @@
     },
     "packages/browsers": {
       "name": "@puppeteer/browsers",
-      "version": "0.2.0",
+      "version": "0.3.0",
       "license": "Apache-2.0",
       "dependencies": {
         "debug": "4.3.4",
@@ -9470,6 +9470,7 @@
       "hasInstallScript": true,
       "license": "Apache-2.0",
       "dependencies": {
+        "@puppeteer/browsers": "0.3.0",
         "cosmiconfig": "8.1.3",
         "https-proxy-agent": "5.0.1",
         "progress": "2.0.3",
@@ -14452,6 +14453,7 @@
     "puppeteer": {
       "version": "file:packages/puppeteer",
       "requires": {
+        "@puppeteer/browsers": "0.3.0",
         "cosmiconfig": "8.1.3",
         "https-proxy-agent": "5.0.1",
         "progress": "2.0.3",

packages/puppeteer/package.json

@@ -81,6 +81,7 @@
       "clean": "if-file-deleted",
       "dependencies": [
         "../puppeteer-core:build",
+        "../browsers:build",
         "generate:package-json"
       ],
       "files": [
@@ -119,6 +120,7 @@
     "https-proxy-agent": "5.0.1",
     "progress": "2.0.3",
     "proxy-from-env": "1.1.0",
-    "puppeteer-core": "19.8.0"
+    "puppeteer-core": "19.8.0",
+    "@puppeteer/browsers": "0.3.0"
   }
 }

packages/puppeteer/src/node/install.ts

@@ -14,13 +14,15 @@
  * limitations under the License.
  */
 
-import https, {RequestOptions} from 'https';
+import {
-import URL from 'url';
+  fetch,
-
+  Browser,
-import createHttpsProxyAgent, {HttpsProxyAgentOptions} from 'https-proxy-agent';
+  resolveBuildId,
-import ProgressBar from 'progress';
+  makeProgressCallback,
-import {getProxyForUrl} from 'proxy-from-env';
+  detectBrowserPlatform,
-import {PuppeteerNode} from 'puppeteer-core/internal/node/PuppeteerNode.js';
+  BrowserPlatform,
+} from '@puppeteer/browsers';
+import {Product} from 'puppeteer-core';
 import {PUPPETEER_REVISIONS} from 'puppeteer-core/internal/revisions.js';
 
 import {getConfiguration} from '../getConfiguration.js';
@@ -29,6 +31,7 @@ import {getConfiguration} from '../getConfiguration.js';
  * @internal
  */
 const supportedProducts = {
+  chromium: 'Chromium',
   chrome: 'Chromium',
   firefox: 'Firefox Nightly',
 } as const;
@@ -37,170 +40,69 @@ const supportedProducts = {
  * @internal
  */
 export async function downloadBrowser(): Promise<void> {
+  overrideProxy();
+
   const configuration = getConfiguration();
   if (configuration.skipDownload) {
     logPolitely('**INFO** Skipping browser download as instructed.');
     return;
   }
 
-  const puppeteer = new PuppeteerNode({configuration, isPuppeteerCore: false});
+  let platform = detectBrowserPlatform();
+  if (!platform) {
+    throw new Error('The current platform is not supported.');
+  }
+
+  // TODO: remove once Mac ARM is enabled by default for Puppeteer https://github.com/puppeteer/puppeteer/issues/9630.
+  if (
+    platform === BrowserPlatform.MAC_ARM &&
+    !configuration.experiments?.macArmChromiumEnabled
+  ) {
+    platform = BrowserPlatform.MAC;
+  }
 
   const product = configuration.defaultProduct!;
-  const browserFetcher = puppeteer.createBrowserFetcher();
+  const browser = productToBrowser(product);
 
-  let revision = configuration.browserRevision;
+  // TODO: PUPPETEER_REVISIONS should use Chrome and not Chromium.
+  const unresolvedBuildId =
+    configuration.browserRevision ||
+    PUPPETEER_REVISIONS[product === 'chrome' ? 'chromium' : 'firefox'] ||
+    'latest';
 
-  if (!revision) {
+  const buildId = await resolveBuildId(browser, platform, unresolvedBuildId);
-    switch (product) {
-      case 'chrome':
-        revision = PUPPETEER_REVISIONS.chromium;
-        break;
-      case 'firefox':
-        revision = PUPPETEER_REVISIONS.firefox;
-        revision = await getFirefoxNightlyVersion();
-        break;
-    }
-  }
 
-  await fetchBinary(revision);
+  try {
-
+    const result = await fetch({
-  function fetchBinary(revision: string) {
+      browser,
-    const revisionInfo = browserFetcher.revisionInfo(revision);
+      cacheDir: configuration.cacheDirectory!,
-
+      platform,
-    // Do nothing if the revision is already downloaded.
+      buildId,
-    if (revisionInfo.local) {
+      downloadProgressCallback: makeProgressCallback(browser, buildId),
-      logPolitely(
-        `${supportedProducts[product]} is already in ${revisionInfo.folderPath}; skipping download.`
-      );
-      return;
-    }
-
-    // Override current environment proxy settings with npm configuration, if any.
-    const NPM_HTTPS_PROXY =
-      process.env['npm_config_https_proxy'] || process.env['npm_config_proxy'];
-    const NPM_HTTP_PROXY =
-      process.env['npm_config_http_proxy'] || process.env['npm_config_proxy'];
-    const NPM_NO_PROXY = process.env['npm_config_no_proxy'];
-
-    if (NPM_HTTPS_PROXY) {
-      process.env['HTTPS_PROXY'] = NPM_HTTPS_PROXY;
-    }
-    if (NPM_HTTP_PROXY) {
-      process.env['HTTP_PROXY'] = NPM_HTTP_PROXY;
-    }
-    if (NPM_NO_PROXY) {
-      process.env['NO_PROXY'] = NPM_NO_PROXY;
-    }
-
-    function onSuccess(localRevisions: string[]): void {
-      logPolitely(
-        `${supportedProducts[product]} (${revisionInfo.revision}) downloaded to ${revisionInfo.folderPath}`
-      );
-      const otherRevisions = localRevisions.filter(revision => {
-        return revision !== revisionInfo.revision;
-      });
-      if (otherRevisions.length) {
-        logPolitely(
-          `Other installed ${
-            supportedProducts[product]
-          } browsers in ${browserFetcher.getDownloadPath()} include: ${otherRevisions.join(
-            ', '
-          )}. Remove old revisions from ${browserFetcher.getDownloadPath()} if you don't need them.`
-        );
-      }
-    }
-
-    function onError(error: Error) {
-      console.error(
-        `ERROR: Failed to set up ${supportedProducts[product]} r${revision}! Set "PUPPETEER_SKIP_DOWNLOAD" env variable to skip download.`
-      );
-      console.error(error);
-      process.exit(1);
-    }
-
-    let progressBar: ProgressBar | null = null;
-    let lastDownloadedBytes = 0;
-    function onProgress(downloadedBytes: number, totalBytes: number) {
-      if (!progressBar) {
-        progressBar = new ProgressBar(
-          `Downloading ${
-            supportedProducts[product]
-          } r${revision} - ${toMegabytes(totalBytes)} [:bar] :percent :etas `,
-          {
-            complete: '=',
-            incomplete: ' ',
-            width: 20,
-            total: totalBytes,
-          }
-        );
-      }
-      const delta = downloadedBytes - lastDownloadedBytes;
-      lastDownloadedBytes = downloadedBytes;
-      progressBar.tick(delta);
-    }
-
-    return browserFetcher
-      .download(revisionInfo.revision, onProgress)
-      .then(() => {
-        return browserFetcher.localRevisions();
-      })
-      .then(onSuccess)
-      .catch(onError);
-  }
-
-  function toMegabytes(bytes: number) {
-    const mb = bytes / 1024 / 1024;
-    return `${Math.round(mb * 10) / 10} Mb`;
-  }
-
-  async function getFirefoxNightlyVersion(): Promise<string> {
-    const firefoxVersionsUrl =
-      'https://product-details.mozilla.org/1.0/firefox_versions.json';
-
-    const proxyURL = getProxyForUrl(firefoxVersionsUrl);
-
-    const requestOptions: RequestOptions = {};
-
-    if (proxyURL) {
-      const parsedProxyURL = URL.parse(proxyURL);
-
-      const proxyOptions = {
-        ...parsedProxyURL,
-        secureProxy: parsedProxyURL.protocol === 'https:',
-      } as HttpsProxyAgentOptions;
-
-      requestOptions.agent = createHttpsProxyAgent(proxyOptions);
-      requestOptions.rejectUnauthorized = false;
-    }
-
-    const promise = new Promise<string>((resolve, reject) => {
-      let data = '';
-      logPolitely(
-        `Requesting latest Firefox Nightly version from ${firefoxVersionsUrl}`
-      );
-      https
-        .get(firefoxVersionsUrl, requestOptions, r => {
-          if (r.statusCode && r.statusCode >= 400) {
-            return reject(new Error(`Got status code ${r.statusCode}`));
-          }
-          r.on('data', chunk => {
-            data += chunk;
-          });
-          r.on('end', () => {
-            try {
-              const versions = JSON.parse(data);
-              return resolve(versions.FIREFOX_NIGHTLY);
-            } catch {
-              return reject(new Error('Firefox version not found'));
-            }
-          });
-        })
-        .on('error', reject);
     });
-    return promise;
+
+    logPolitely(
+      `${supportedProducts[product]} (${result.buildId}) downloaded to ${result.path}`
+    );
+  } catch (error) {
+    console.error(
+      `ERROR: Failed to set up ${supportedProducts[product]} r${buildId}! Set "PUPPETEER_SKIP_DOWNLOAD" env variable to skip download.`
+    );
+    console.error(error);
+    process.exit(1);
   }
 }
 
+function productToBrowser(product?: Product) {
+  switch (product) {
+    case 'chrome':
+      return Browser.CHROMIUM;
+    case 'firefox':
+      return Browser.FIREFOX;
+  }
+  return Browser.CHROMIUM;
+}
+
 /**
  * @internal
  */
@@ -213,3 +115,25 @@ function logPolitely(toBeLogged: unknown): void {
     console.log(toBeLogged);
   }
 }
+
+/**
+ * @internal
+ */
+function overrideProxy() {
+  // Override current environment proxy settings with npm configuration, if any.
+  const NPM_HTTPS_PROXY =
+    process.env['npm_config_https_proxy'] || process.env['npm_config_proxy'];
+  const NPM_HTTP_PROXY =
+    process.env['npm_config_http_proxy'] || process.env['npm_config_proxy'];
+  const NPM_NO_PROXY = process.env['npm_config_no_proxy'];
+
+  if (NPM_HTTPS_PROXY) {
+    process.env['HTTPS_PROXY'] = NPM_HTTPS_PROXY;
+  }
+  if (NPM_HTTP_PROXY) {
+    process.env['HTTP_PROXY'] = NPM_HTTP_PROXY;
+  }
+  if (NPM_NO_PROXY) {
+    process.env['NO_PROXY'] = NPM_NO_PROXY;
+  }
+}

test/TestExpectations.json

@@ -305,12 +305,6 @@
     "parameters": ["webDriverBiDi"],
     "expectations": ["FAIL", "TIMEOUT"]
   },
-  {
-    "testIdPattern": "[navigation.spec] navigation Page.goto should not leak listeners during navigation of 11 pages",
-    "platforms": ["darwin"],
-    "parameters": ["chrome", "webDriverBiDi"],
-    "expectations": ["PASS", "TIMEOUT"]
-  },
   {
     "testIdPattern": "[navigation.spec] navigation Page.goto should not throw an error for a 404 response with an empty body",
     "platforms": ["darwin", "linux", "win32"],
@@ -1175,6 +1169,12 @@
     "parameters": ["chrome", "webDriverBiDi"],
     "expectations": ["FAIL", "PASS", "TIMEOUT"]
   },
+  {
+    "testIdPattern": "[navigation.spec] navigation Page.goto should not leak listeners during navigation of 11 pages",
+    "platforms": ["darwin"],
+    "parameters": ["chrome", "webDriverBiDi"],
+    "expectations": ["PASS", "TIMEOUT"]
+  },
   {
     "testIdPattern": "[navigation.spec] navigation Page.goto should send referer",
     "platforms": ["darwin", "linux", "win32"],

test/installation/package.json

@@ -26,10 +26,11 @@
       ]
     },
     "build:packages": {
-      "command": "npm pack --workspace puppeteer --workspace puppeteer-core",
+      "command": "npm pack --workspace puppeteer --workspace puppeteer-core --workspace @puppeteer/browsers",
       "dependencies": [
         "../../packages/puppeteer:build",
-        "../../packages/puppeteer-core:build"
+        "../../packages/puppeteer-core:build",
+        "../../packages/browsers:build"
       ],
       "files": [],
       "output": [

test/installation/src/constants.ts

@@ -22,6 +22,9 @@ import glob from 'glob';
 export const PUPPETEER_CORE_PACKAGE_PATH = resolve(
   glob.sync('puppeteer-core-*.tgz')[0]!
 );
+export const PUPPETEER_BROWSERS_PACKAGE_PATH = resolve(
+  glob.sync('puppeteer-browsers-[0-9]*.tgz')[0]!
+);
 export const PUPPETEER_PACKAGE_PATH = resolve(
   glob.sync('puppeteer-[0-9]*.tgz')[0]!
 );

test/installation/src/describeInstallation.ts

@@ -22,6 +22,7 @@ import {join} from 'path';
 import {
   PUPPETEER_CORE_PACKAGE_PATH,
   PUPPETEER_PACKAGE_PATH,
+  PUPPETEER_BROWSERS_PACKAGE_PATH,
 } from './constants.js';
 import {execFile} from './util.js';
 
@@ -78,6 +79,8 @@ export const describeInstallation = (
             return PUPPETEER_PACKAGE_PATH;
           case 'puppeteer-core':
             return PUPPETEER_CORE_PACKAGE_PATH;
+          case '@puppeteer/browsers':
+            return PUPPETEER_BROWSERS_PACKAGE_PATH;
           default:
             return module;
         }
@@ -102,7 +105,7 @@ export const describeInstallation = (
 
     after(async () => {
       if (process.env['KEEP_SANDBOX']) {
-        await rm(sandbox, {recursive: true, force: true});
+        await rm(sandbox, {recursive: true, force: true, maxRetries: 5});
       }
     });
 

test/installation/src/puppeteer-configuration.spec.ts

@@ -24,7 +24,7 @@ import {readAsset} from './util.js';
 describeInstallation(
   '`puppeteer` with configuration',
   {
-    dependencies: ['puppeteer-core', 'puppeteer'],
+    dependencies: ['puppeteer-core', '@puppeteer/browsers', 'puppeteer'],
     before: async cwd => {
       await writeFile(
         join(cwd, '.puppeteerrc.cjs'),

test/installation/src/puppeteer-firefox.spec.ts

@@ -24,7 +24,7 @@ import {readAsset} from './util.js';
 describeInstallation(
   '`puppeteer` with Firefox',
   {
-    dependencies: ['puppeteer-core', 'puppeteer'],
+    dependencies: ['@puppeteer/browsers', 'puppeteer-core', 'puppeteer'],
     env: cwd => {
       return {
         PUPPETEER_CACHE_DIR: join(cwd, '.cache', 'puppeteer'),

test/installation/src/puppeteer-webpack.spec.ts

@@ -23,7 +23,7 @@ import {execFile, readAsset} from './util.js';
 describeInstallation(
   '`puppeteer` with Webpack',
   {
-    dependencies: ['puppeteer-core', 'puppeteer'],
+    dependencies: ['@puppeteer/browsers', 'puppeteer-core', 'puppeteer'],
     devDependencies: ['webpack', 'webpack-cli'],
     env: cwd => {
       return {

test/installation/src/puppeteer.spec.ts

@@ -24,7 +24,7 @@ import {readAsset} from './util.js';
 describeInstallation(
   '`puppeteer`',
   {
-    dependencies: ['puppeteer-core', 'puppeteer'],
+    dependencies: ['@puppeteer/browsers', 'puppeteer-core', 'puppeteer'],
     env: cwd => {
       return {
         PUPPETEER_CACHE_DIR: join(cwd, '.cache', 'puppeteer'),