refactor: use @puppeteer/browsers for install (#9898)

Co-authored-by: Nikolay Vitkov <34244704+Lightning00Blade@users.noreply.github.com>
2024-06-14 14:02:48 +00:00 · 2023-03-28 11:27:50 +02:00 · 2023-03-28 11:27:50 +02:00 · 3936600ba9
commit 3936600ba9
parent a88322e947
14 changed files with 115 additions and 178 deletions
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@ -143,7 +143,7 @@ jobs:
        uses: actions/cache@v3
        with:
          path: ~/.cache/puppeteer/chrome
-          key: ${{ runner.os }}-chromium-${{ hashFiles('packages/puppeteer-core/src/revisions.ts') }}
+          key: ${{ runner.os }}-chromium-${{ hashFiles('packages/puppeteer-core/src/revisions.ts') }}-${{ hashFiles('packages/puppeteer/src/node/install.ts') }}
      - name: Install Chromium
        run: npm run postinstall
      - name: Tests types
@ -211,7 +211,7 @@ jobs:
        uses: actions/cache@v3
        with:
          path: ~/.cache/puppeteer/firefox
-          key: ${{ runner.os }}-firefox-${{ hashFiles('packages/puppeteer-core/src/revisions.ts') }}
+          key: ${{ runner.os }}-firefox-${{ hashFiles('packages/puppeteer-core/src/revisions.ts') }}-${{ hashFiles('packages/puppeteer/src/node/install.ts') }}
      - name: Install Firefox
        env:
          PUPPETEER_PRODUCT: firefox
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@ -17,11 +17,11 @@ USER pptruser

 WORKDIR /home/pptruser

-COPY puppeteer-latest.tgz puppeteer-core-latest.tgz ./
+COPY puppeteer-browsers-latest.tgz puppeteer-latest.tgz puppeteer-core-latest.tgz ./

-# Install puppeteer and puppeteer-core into /home/pptruser/node_modules.
-RUN npm i ./puppeteer-core-latest.tgz ./puppeteer-latest.tgz \
-    && rm ./puppeteer-core-latest.tgz ./puppeteer-latest.tgz \
+# Install @puppeteer/browsers, puppeteer and puppeteer-core into /home/pptruser/node_modules.
+RUN npm i ./puppeteer-browsers-latest.tgz ./puppeteer-core-latest.tgz ./puppeteer-latest.tgz \
+    && rm ./puppeteer-browsers-latest.tgz ./puppeteer-core-latest.tgz ./puppeteer-latest.tgz \
    && (node -e "require('child_process').execSync(require('puppeteer').executablePath() + ' --credits', {stdio: 'inherit'})" > THIRD_PARTY_NOTICES)

 CMD ["google-chrome-stable"]
--- a/docker/pack.sh
+++ b/docker/pack.sh
@ -7,10 +7,12 @@ set -e

 cd docker

-npm pack --workspace puppeteer --workspace puppeteer-core --pack-destination .
+npm pack --workspace puppeteer --workspace puppeteer-core --workspace @puppeteer/browsers --pack-destination .

 rm -f puppeteer-core-latest.tgz
 rm -f puppeteer-latest.tgz
+rm -f puppeteer-browsers-latest.tgz

 mv puppeteer-core-*.tgz puppeteer-core-latest.tgz
+mv puppeteer-browsers-*.tgz puppeteer-browsers-latest.tgz
 mv puppeteer-[0-9]*.tgz puppeteer-latest.tgz
--- a/package-lock.json
+++ b/package-lock.json
@ -9317,7 +9317,7 @@
    },
    "packages/browsers": {
      "name": "@puppeteer/browsers",
-      "version": "0.2.0",
+      "version": "0.3.0",
      "license": "Apache-2.0",
      "dependencies": {
        "debug": "4.3.4",
@ -9470,6 +9470,7 @@
      "hasInstallScript": true,
      "license": "Apache-2.0",
      "dependencies": {
+        "@puppeteer/browsers": "0.3.0",
        "cosmiconfig": "8.1.3",
        "https-proxy-agent": "5.0.1",
        "progress": "2.0.3",
@ -14452,6 +14453,7 @@
    "puppeteer": {
      "version": "file:packages/puppeteer",
      "requires": {
+        "@puppeteer/browsers": "0.3.0",
        "cosmiconfig": "8.1.3",
        "https-proxy-agent": "5.0.1",
        "progress": "2.0.3",
--- a/packages/puppeteer/package.json
+++ b/packages/puppeteer/package.json
@ -81,6 +81,7 @@
      "clean": "if-file-deleted",
      "dependencies": [
        "../puppeteer-core:build",
+        "../browsers:build",
        "generate:package-json"
      ],
      "files": [
@ -119,6 +120,7 @@
    "https-proxy-agent": "5.0.1",
    "progress": "2.0.3",
    "proxy-from-env": "1.1.0",
-    "puppeteer-core": "19.8.0"
+    "puppeteer-core": "19.8.0",
+    "@puppeteer/browsers": "0.3.0"
  }
 }
--- a/packages/puppeteer/src/node/install.ts
+++ b/packages/puppeteer/src/node/install.ts
@ -14,13 +14,15 @@
 * limitations under the License.
 */

-import https, {RequestOptions} from 'https';
-import URL from 'url';
-
-import createHttpsProxyAgent, {HttpsProxyAgentOptions} from 'https-proxy-agent';
-import ProgressBar from 'progress';
-import {getProxyForUrl} from 'proxy-from-env';
-import {PuppeteerNode} from 'puppeteer-core/internal/node/PuppeteerNode.js';
+import {
+  fetch,
+  Browser,
+  resolveBuildId,
+  makeProgressCallback,
+  detectBrowserPlatform,
+  BrowserPlatform,
+} from '@puppeteer/browsers';
+import {Product} from 'puppeteer-core';
 import {PUPPETEER_REVISIONS} from 'puppeteer-core/internal/revisions.js';

 import {getConfiguration} from '../getConfiguration.js';
@ -29,6 +31,7 @@ import {getConfiguration} from '../getConfiguration.js';
 * @internal
 */
 const supportedProducts = {
+  chromium: 'Chromium',
  chrome: 'Chromium',
  firefox: 'Firefox Nightly',
 } as const;
@ -37,44 +40,86 @@ const supportedProducts = {
 * @internal
 */
 export async function downloadBrowser(): Promise<void> {
+  overrideProxy();
+
  const configuration = getConfiguration();
  if (configuration.skipDownload) {
    logPolitely('**INFO** Skipping browser download as instructed.');
    return;
  }

-  const puppeteer = new PuppeteerNode({configuration, isPuppeteerCore: false});
+  let platform = detectBrowserPlatform();
+  if (!platform) {
+    throw new Error('The current platform is not supported.');
+  }
+
+  // TODO: remove once Mac ARM is enabled by default for Puppeteer https://github.com/puppeteer/puppeteer/issues/9630.
+  if (
+    platform === BrowserPlatform.MAC_ARM &&
+    !configuration.experiments?.macArmChromiumEnabled
+  ) {
+    platform = BrowserPlatform.MAC;
+  }

  const product = configuration.defaultProduct!;
-  const browserFetcher = puppeteer.createBrowserFetcher();
+  const browser = productToBrowser(product);

-  let revision = configuration.browserRevision;
+  // TODO: PUPPETEER_REVISIONS should use Chrome and not Chromium.
+  const unresolvedBuildId =
+    configuration.browserRevision ||
+    PUPPETEER_REVISIONS[product === 'chrome' ? 'chromium' : 'firefox'] ||
+    'latest';

-  if (!revision) {
+  const buildId = await resolveBuildId(browser, platform, unresolvedBuildId);
+
+  try {
+    const result = await fetch({
+      browser,
+      cacheDir: configuration.cacheDirectory!,
+      platform,
+      buildId,
+      downloadProgressCallback: makeProgressCallback(browser, buildId),
+    });
+
+    logPolitely(
+      `${supportedProducts[product]} (${result.buildId}) downloaded to ${result.path}`
+    );
+  } catch (error) {
+    console.error(
+      `ERROR: Failed to set up ${supportedProducts[product]} r${buildId}! Set "PUPPETEER_SKIP_DOWNLOAD" env variable to skip download.`
+    );
+    console.error(error);
+    process.exit(1);
+  }
+}
+
+function productToBrowser(product?: Product) {
  switch (product) {
    case 'chrome':
-        revision = PUPPETEER_REVISIONS.chromium;
-        break;
+      return Browser.CHROMIUM;
    case 'firefox':
-        revision = PUPPETEER_REVISIONS.firefox;
-        revision = await getFirefoxNightlyVersion();
-        break;
+      return Browser.FIREFOX;
+  }
+  return Browser.CHROMIUM;
+}
+
+/**
+ * @internal
+ */
+function logPolitely(toBeLogged: unknown): void {
+  const logLevel = process.env['npm_config_loglevel'] || '';
+  const logLevelDisplay = ['silent', 'error', 'warn'].indexOf(logLevel) > -1;
+
+  // eslint-disable-next-line no-console
+  if (!logLevelDisplay) {
+    console.log(toBeLogged);
  }
 }

-  await fetchBinary(revision);
-
-  function fetchBinary(revision: string) {
-    const revisionInfo = browserFetcher.revisionInfo(revision);
-
-    // Do nothing if the revision is already downloaded.
-    if (revisionInfo.local) {
-      logPolitely(
-        `${supportedProducts[product]} is already in ${revisionInfo.folderPath}; skipping download.`
-      );
-      return;
-    }
-
+/**
+ * @internal
+ */
+function overrideProxy() {
  // Override current environment proxy settings with npm configuration, if any.
  const NPM_HTTPS_PROXY =
    process.env['npm_config_https_proxy'] || process.env['npm_config_proxy'];
@ -91,125 +136,4 @@ export async function downloadBrowser(): Promise<void> {
  if (NPM_NO_PROXY) {
    process.env['NO_PROXY'] = NPM_NO_PROXY;
  }
-
-    function onSuccess(localRevisions: string[]): void {
-      logPolitely(
-        `${supportedProducts[product]} (${revisionInfo.revision}) downloaded to ${revisionInfo.folderPath}`
-      );
-      const otherRevisions = localRevisions.filter(revision => {
-        return revision !== revisionInfo.revision;
-      });
-      if (otherRevisions.length) {
-        logPolitely(
-          `Other installed ${
-            supportedProducts[product]
-          } browsers in ${browserFetcher.getDownloadPath()} include: ${otherRevisions.join(
-            ', '
-          )}. Remove old revisions from ${browserFetcher.getDownloadPath()} if you don't need them.`
-        );
-      }
-    }
-
-    function onError(error: Error) {
-      console.error(
-        `ERROR: Failed to set up ${supportedProducts[product]} r${revision}! Set "PUPPETEER_SKIP_DOWNLOAD" env variable to skip download.`
-      );
-      console.error(error);
-      process.exit(1);
-    }
-
-    let progressBar: ProgressBar | null = null;
-    let lastDownloadedBytes = 0;
-    function onProgress(downloadedBytes: number, totalBytes: number) {
-      if (!progressBar) {
-        progressBar = new ProgressBar(
-          `Downloading ${
-            supportedProducts[product]
-          } r${revision} - ${toMegabytes(totalBytes)} [:bar] :percent :etas `,
-          {
-            complete: '=',
-            incomplete: ' ',
-            width: 20,
-            total: totalBytes,
-          }
-        );
-      }
-      const delta = downloadedBytes - lastDownloadedBytes;
-      lastDownloadedBytes = downloadedBytes;
-      progressBar.tick(delta);
-    }
-
-    return browserFetcher
-      .download(revisionInfo.revision, onProgress)
-      .then(() => {
-        return browserFetcher.localRevisions();
-      })
-      .then(onSuccess)
-      .catch(onError);
-  }
-
-  function toMegabytes(bytes: number) {
-    const mb = bytes / 1024 / 1024;
-    return `${Math.round(mb * 10) / 10} Mb`;
-  }
-
-  async function getFirefoxNightlyVersion(): Promise<string> {
-    const firefoxVersionsUrl =
-      'https://product-details.mozilla.org/1.0/firefox_versions.json';
-
-    const proxyURL = getProxyForUrl(firefoxVersionsUrl);
-
-    const requestOptions: RequestOptions = {};
-
-    if (proxyURL) {
-      const parsedProxyURL = URL.parse(proxyURL);
-
-      const proxyOptions = {
-        ...parsedProxyURL,
-        secureProxy: parsedProxyURL.protocol === 'https:',
-      } as HttpsProxyAgentOptions;
-
-      requestOptions.agent = createHttpsProxyAgent(proxyOptions);
-      requestOptions.rejectUnauthorized = false;
-    }
-
-    const promise = new Promise<string>((resolve, reject) => {
-      let data = '';
-      logPolitely(
-        `Requesting latest Firefox Nightly version from ${firefoxVersionsUrl}`
-      );
-      https
-        .get(firefoxVersionsUrl, requestOptions, r => {
-          if (r.statusCode && r.statusCode >= 400) {
-            return reject(new Error(`Got status code ${r.statusCode}`));
-          }
-          r.on('data', chunk => {
-            data += chunk;
-          });
-          r.on('end', () => {
-            try {
-              const versions = JSON.parse(data);
-              return resolve(versions.FIREFOX_NIGHTLY);
-            } catch {
-              return reject(new Error('Firefox version not found'));
-            }
-          });
-        })
-        .on('error', reject);
-    });
-    return promise;
-  }
-}
-
-/**
- * @internal
- */
-function logPolitely(toBeLogged: unknown): void {
-  const logLevel = process.env['npm_config_loglevel'] || '';
-  const logLevelDisplay = ['silent', 'error', 'warn'].indexOf(logLevel) > -1;
-
-  // eslint-disable-next-line no-console
-  if (!logLevelDisplay) {
-    console.log(toBeLogged);
-  }
 }
--- a/test/TestExpectations.json
+++ b/test/TestExpectations.json
@ -305,12 +305,6 @@
    "parameters": ["webDriverBiDi"],
    "expectations": ["FAIL", "TIMEOUT"]
  },
-  {
-    "testIdPattern": "[navigation.spec] navigation Page.goto should not leak listeners during navigation of 11 pages",
-    "platforms": ["darwin"],
-    "parameters": ["chrome", "webDriverBiDi"],
-    "expectations": ["PASS", "TIMEOUT"]
-  },
  {
    "testIdPattern": "[navigation.spec] navigation Page.goto should not throw an error for a 404 response with an empty body",
    "platforms": ["darwin", "linux", "win32"],
@ -1175,6 +1169,12 @@
    "parameters": ["chrome", "webDriverBiDi"],
    "expectations": ["FAIL", "PASS", "TIMEOUT"]
  },
+  {
+    "testIdPattern": "[navigation.spec] navigation Page.goto should not leak listeners during navigation of 11 pages",
+    "platforms": ["darwin"],
+    "parameters": ["chrome", "webDriverBiDi"],
+    "expectations": ["PASS", "TIMEOUT"]
+  },
  {
    "testIdPattern": "[navigation.spec] navigation Page.goto should send referer",
    "platforms": ["darwin", "linux", "win32"],
--- a/test/installation/package.json
+++ b/test/installation/package.json
@ -26,10 +26,11 @@
      ]
    },
    "build:packages": {
-      "command": "npm pack --workspace puppeteer --workspace puppeteer-core",
+      "command": "npm pack --workspace puppeteer --workspace puppeteer-core --workspace @puppeteer/browsers",
      "dependencies": [
        "../../packages/puppeteer:build",
-        "../../packages/puppeteer-core:build"
+        "../../packages/puppeteer-core:build",
+        "../../packages/browsers:build"
      ],
      "files": [],
      "output": [
--- a/test/installation/src/constants.ts
+++ b/test/installation/src/constants.ts
@ -22,6 +22,9 @@ import glob from 'glob';
 export const PUPPETEER_CORE_PACKAGE_PATH = resolve(
  glob.sync('puppeteer-core-*.tgz')[0]!
 );
+export const PUPPETEER_BROWSERS_PACKAGE_PATH = resolve(
+  glob.sync('puppeteer-browsers-[0-9]*.tgz')[0]!
+);
 export const PUPPETEER_PACKAGE_PATH = resolve(
  glob.sync('puppeteer-[0-9]*.tgz')[0]!
 );
--- a/test/installation/src/describeInstallation.ts
+++ b/test/installation/src/describeInstallation.ts
@ -22,6 +22,7 @@ import {join} from 'path';
 import {
  PUPPETEER_CORE_PACKAGE_PATH,
  PUPPETEER_PACKAGE_PATH,
+  PUPPETEER_BROWSERS_PACKAGE_PATH,
 } from './constants.js';
 import {execFile} from './util.js';

@ -78,6 +79,8 @@ export const describeInstallation = (
            return PUPPETEER_PACKAGE_PATH;
          case 'puppeteer-core':
            return PUPPETEER_CORE_PACKAGE_PATH;
+          case '@puppeteer/browsers':
+            return PUPPETEER_BROWSERS_PACKAGE_PATH;
          default:
            return module;
        }
@ -102,7 +105,7 @@ export const describeInstallation = (

    after(async () => {
      if (process.env['KEEP_SANDBOX']) {
-        await rm(sandbox, {recursive: true, force: true});
+        await rm(sandbox, {recursive: true, force: true, maxRetries: 5});
      }
    });

--- a/test/installation/src/puppeteer-configuration.spec.ts
+++ b/test/installation/src/puppeteer-configuration.spec.ts
@ -24,7 +24,7 @@ import {readAsset} from './util.js';
 describeInstallation(
  '`puppeteer` with configuration',
  {
-    dependencies: ['puppeteer-core', 'puppeteer'],
+    dependencies: ['puppeteer-core', '@puppeteer/browsers', 'puppeteer'],
    before: async cwd => {
      await writeFile(
        join(cwd, '.puppeteerrc.cjs'),
--- a/test/installation/src/puppeteer-firefox.spec.ts
+++ b/test/installation/src/puppeteer-firefox.spec.ts
@ -24,7 +24,7 @@ import {readAsset} from './util.js';
 describeInstallation(
  '`puppeteer` with Firefox',
  {
-    dependencies: ['puppeteer-core', 'puppeteer'],
+    dependencies: ['@puppeteer/browsers', 'puppeteer-core', 'puppeteer'],
    env: cwd => {
      return {
        PUPPETEER_CACHE_DIR: join(cwd, '.cache', 'puppeteer'),
--- a/test/installation/src/puppeteer-webpack.spec.ts
+++ b/test/installation/src/puppeteer-webpack.spec.ts
@ -23,7 +23,7 @@ import {execFile, readAsset} from './util.js';
 describeInstallation(
  '`puppeteer` with Webpack',
  {
-    dependencies: ['puppeteer-core', 'puppeteer'],
+    dependencies: ['@puppeteer/browsers', 'puppeteer-core', 'puppeteer'],
    devDependencies: ['webpack', 'webpack-cli'],
    env: cwd => {
      return {
--- a/test/installation/src/puppeteer.spec.ts
+++ b/test/installation/src/puppeteer.spec.ts
@ -24,7 +24,7 @@ import {readAsset} from './util.js';
 describeInstallation(
  '`puppeteer`',
  {
-    dependencies: ['puppeteer-core', 'puppeteer'],
+    dependencies: ['@puppeteer/browsers', 'puppeteer-core', 'puppeteer'],
    env: cwd => {
      return {
        PUPPETEER_CACHE_DIR: join(cwd, '.cache', 'puppeteer'),