chore: test server for browsers (#9974)

Co-authored-by: Nikolay Vitkov <34244704+Lightning00Blade@users.noreply.github.com>

.github/workflows/ci.yml

@@ -399,6 +399,11 @@ jobs:
         run: npm ci
         env:
           PUPPETEER_SKIP_DOWNLOAD: true
+      - name: Setup cache for browser binaries
+        uses: actions/cache@v3
+        with:
+          path: packages/browsers/test/cache
+          key: browsers-${{ hashFiles('packages/browsers/tools/downloadTestBrowsers.mjs') }}-${{ hashFiles('packages/browsers/test/src/versions.ts') }}
       - name: Run tests
         run: npm run test --workspace @puppeteer/browsers
 

packages/browsers/.gitignore

@@ -0,0 +1 @@
+test/cache

packages/browsers/.mocharc.cjs

@@ -3,4 +3,5 @@ module.exports = {
   spec: 'test/build/**/*.spec.js',
   exit: !!process.env.CI,
   reporter: 'spec',
+  timeout: 10_000,
 };

packages/browsers/package.json

@@ -22,13 +22,27 @@
   },
   "wireit": {
     "build": {
-      "command": "tsc -b && chmod a+x lib/cjs/main-cli.js lib/esm/main-cli.js && echo '{\"type\":\"module\"}' > lib/esm/package.json",
+      "command": "tsc -b && chmod a+x lib/cjs/main-cli.js lib/esm/main-cli.js",
       "files": [
         "src/**/*.ts",
         "tsconfig.json"
       ],
+      "clean": "if-file-deleted",
       "output": [
-        "lib/**"
+        "lib/**",
+        "!lib/esm/package.json"
+      ],
+      "dependencies": [
+        "generate:package-json"
+      ]
+    },
+    "generate:package-json": {
+      "command": "tsx ../../tools/generate_module_package_json.ts lib/esm/package.json",
+      "files": [
+        "../../tools/generate_module_package_json.ts"
+      ],
+      "output": [
+        "lib/esm/package.json"
       ]
     },
     "build:test": {
@@ -41,11 +55,12 @@
         "test/build/**"
       ],
       "dependencies": [
-        "build"
+        "build",
+        "../testserver:build"
       ]
     },
     "test": {
-      "command": "mocha",
+      "command": "node tools/downloadTestBrowsers.mjs && mocha",
       "files": [
         ".mocharc.cjs"
       ],

packages/browsers/src/CLI.ts

@@ -44,6 +44,7 @@ type InstallArgs = {
   };
   path?: string;
   platform?: BrowserPlatform;
+  baseUrl?: string;
 };
 
 type LaunchArgs = {
@@ -116,6 +117,10 @@ export class CLI {
           this.#defineBrowserParameter(yargs);
           this.#definePlatformParameter(yargs);
           this.#definePathParameter(yargs);
+          yargs.option('base-url', {
+            type: 'string',
+            desc: 'Base URL to download from',
+          });
           yargs.example(
             '$0 install chrome',
             'Install the latest available build of the Chrome browser.'
@@ -161,6 +166,7 @@ export class CLI {
               args.browser.name,
               args.browser.buildId
             ),
+            baseUrl: args.baseUrl,
           });
           console.log(
             `${args.browser.name}@${

packages/browsers/src/browser-data/browser-data.ts

@@ -31,6 +31,12 @@ export const downloadUrls = {
   [Browser.FIREFOX]: firefox.resolveDownloadUrl,
 };
 
+export const downloadPaths = {
+  [Browser.CHROME]: chrome.resolveDownloadPath,
+  [Browser.CHROMIUM]: chromium.resolveDownloadPath,
+  [Browser.FIREFOX]: firefox.resolveDownloadPath,
+};
+
 export const executablePathByBrowser = {
   [Browser.CHROME]: chrome.relativeExecutablePath,
   [Browser.CHROMIUM]: chromium.relativeExecutablePath,

packages/browsers/src/browser-data/chrome.ts

@@ -55,9 +55,14 @@ export function resolveDownloadUrl(
   buildId: string,
   baseUrl = 'https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing'
 ): string {
-  return `${baseUrl}/${buildId}/${folder(platform)}/chrome-${folder(
-    platform
-  )}.zip`;
+  return `${baseUrl}/${resolveDownloadPath(platform, buildId).join('/')}`;
+}
+
+export function resolveDownloadPath(
+  platform: BrowserPlatform,
+  buildId: string
+): string[] {
+  return [buildId, folder(platform), `chrome-${folder(platform)}.zip`];
 }
 
 export function relativeExecutablePath(

packages/browsers/src/browser-data/chromium.ts

@@ -56,10 +56,14 @@ export function resolveDownloadUrl(
   buildId: string,
   baseUrl = 'https://storage.googleapis.com/chromium-browser-snapshots'
 ): string {
-  return `${baseUrl}/${folder(platform)}/${buildId}/${archive(
-    platform,
-    buildId
-  )}.zip`;
+  return `${baseUrl}/${resolveDownloadPath(platform, buildId).join('/')}`;
+}
+
+export function resolveDownloadPath(
+  platform: BrowserPlatform,
+  buildId: string
+): string[] {
+  return [folder(platform), buildId, `${archive(platform, buildId)}.zip`];
 }
 
 export function relativeExecutablePath(

packages/browsers/src/browser-data/firefox.ts

@@ -39,7 +39,14 @@ export function resolveDownloadUrl(
   buildId: string,
   baseUrl = 'https://archive.mozilla.org/pub/firefox/nightly/latest-mozilla-central'
 ): string {
-  return `${baseUrl}/${archive(platform, buildId)}`;
+  return `${baseUrl}/${resolveDownloadPath(platform, buildId).join('/')}`;
+}
+
+export function resolveDownloadPath(
+  platform: BrowserPlatform,
+  buildId: string
+): string[] {
+  return [archive(platform, buildId)];
 }
 
 export function relativeExecutablePath(

packages/browsers/src/fetch.ts

@@ -73,6 +73,12 @@ export interface Options {
    *
    */
   baseUrl?: string;
+  /**
+   * Whether to unpack and install browser archives.
+   *
+   * @defaultValue `true`
+   */
+  install?: boolean;
 }
 
 export type InstalledBrowser = {
@@ -84,6 +90,7 @@ export type InstalledBrowser = {
 
 export async function fetch(options: Options): Promise<InstalledBrowser> {
   options.platform ??= detectBrowserPlatform();
+  options.install ??= true;
   if (!options.platform) {
     throw new Error(
       `Cannot download a binary for the provided platform: ${os.platform()} (${os.arch()})`
@@ -103,6 +110,26 @@ export async function fetch(options: Options): Promise<InstalledBrowser> {
   if (!existsSync(browserRoot)) {
     await mkdir(browserRoot, {recursive: true});
   }
+
+  if (!options.install) {
+    if (existsSync(archivePath)) {
+      return {
+        path: archivePath,
+        browser: options.browser,
+        platform: options.platform,
+        buildId: options.buildId,
+      };
+    }
+    debugFetch(`Downloading binary from ${url}`);
+    await downloadFile(url, archivePath, options.downloadProgressCallback);
+    return {
+      path: archivePath,
+      browser: options.browser,
+      platform: options.platform,
+      buildId: options.buildId,
+    };
+  }
+
   const outputPath = structure.installationDir(
     options.browser,
     options.platform,

packages/browsers/test/src/chrome/cli.spec.ts

@@ -20,12 +20,18 @@ import os from 'os';
 import path from 'path';
 
 import {CLI} from '../../../lib/cjs/CLI.js';
-import {createMockedReadlineInterface} from '../utils.js';
+import {
+  createMockedReadlineInterface,
+  setupTestServer,
+  getServerUrl,
+} from '../utils.js';
 import {testChromeBuildId} from '../versions.js';
 
 describe('Chrome CLI', function () {
   this.timeout(90000);
 
+  setupTestServer();
+
   let tmpDir = '/tmp/puppeteer-browsers-test';
 
   beforeEach(() => {
@@ -38,6 +44,7 @@ describe('Chrome CLI', function () {
       '@puppeteer/browsers',
       'clear',
       `--path=${tmpDir}`,
+      `--base-url=${getServerUrl()}`,
     ]);
   });
 
@@ -49,6 +56,7 @@ describe('Chrome CLI', function () {
       `chrome@${testChromeBuildId}`,
       `--path=${tmpDir}`,
       '--platform=linux',
+      `--base-url=${getServerUrl()}`,
     ]);
     assert.ok(
       fs.existsSync(
@@ -90,6 +98,7 @@ describe('Chrome CLI', function () {
       `chrome@latest`,
       `--path=${tmpDir}`,
       '--platform=linux',
+      `--base-url=${getServerUrl()}`,
     ]);
   });
 });

packages/browsers/test/src/chrome/fetch.spec.ts

@@ -28,6 +28,7 @@ import {
   BrowserPlatform,
   Cache,
 } from '../../../lib/cjs/main.js';
+import {getServerUrl, setupTestServer} from '../utils.js';
 import {testChromeBuildId} from '../versions.js';
 
 /**
@@ -35,6 +36,8 @@ import {testChromeBuildId} from '../versions.js';
  * so it requires the network access.
  */
 describe('Chrome fetch', () => {
+  setupTestServer();
+
   let tmpDir = '/tmp/puppeteer-browsers-test';
 
   beforeEach(() => {
@@ -52,6 +55,7 @@ describe('Chrome fetch', () => {
         browser: Browser.CHROME,
         platform: BrowserPlatform.LINUX,
         buildId: testChromeBuildId,
+        baseUrl: getServerUrl(),
       })
     );
   });
@@ -63,6 +67,7 @@ describe('Chrome fetch', () => {
         browser: Browser.CHROME,
         platform: BrowserPlatform.LINUX,
         buildId: 'unknown',
+        baseUrl: getServerUrl(),
       }),
       false
     );
@@ -81,6 +86,7 @@ describe('Chrome fetch', () => {
       browser: Browser.CHROME,
       platform: BrowserPlatform.LINUX,
       buildId: testChromeBuildId,
+      baseUrl: getServerUrl(),
     });
     assert.strictEqual(browser.path, expectedOutputPath);
     assert.ok(fs.existsSync(expectedOutputPath));
@@ -90,6 +96,7 @@ describe('Chrome fetch', () => {
       browser: Browser.CHROME,
       platform: BrowserPlatform.LINUX,
       buildId: testChromeBuildId,
+      baseUrl: getServerUrl(),
     });
     assert.strictEqual(browser.path, expectedOutputPath);
     assert.ok(fs.existsSync(expectedOutputPath));
@@ -159,11 +166,12 @@ describe('Chrome fetch', () => {
           browser: Browser.CHROME,
           platform: BrowserPlatform.LINUX,
           buildId: testChromeBuildId,
+          baseUrl: getServerUrl(),
         }),
         true
       );
       assert.deepStrictEqual(proxiedRequestUrls, [
-        'https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/113.0.5672.0/linux64/chrome-linux64.zip',
+        getServerUrl() + '/113.0.5672.0/linux64/chrome-linux64.zip',
       ]);
     });
 
@@ -180,11 +188,12 @@ describe('Chrome fetch', () => {
         browser: Browser.CHROME,
         platform: BrowserPlatform.LINUX,
         buildId: testChromeBuildId,
+        baseUrl: getServerUrl(),
       });
       assert.strictEqual(browser.path, expectedOutputPath);
       assert.ok(fs.existsSync(expectedOutputPath));
       assert.deepStrictEqual(proxiedRequestUrls, [
-        'https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/113.0.5672.0/linux64/chrome-linux64.zip',
+        getServerUrl() + '/113.0.5672.0/linux64/chrome-linux64.zip',
       ]);
     });
   });

packages/browsers/test/src/chrome/launcher.spec.ts

@@ -28,6 +28,7 @@ import {
   BrowserPlatform,
   Cache,
 } from '../../../lib/cjs/main.js';
+import {getServerUrl, setupTestServer} from '../utils.js';
 import {testChromeBuildId} from '../versions.js';
 
 describe('Chrome', () => {
@@ -44,6 +45,8 @@ describe('Chrome', () => {
   });
 
   describe('launcher', function () {
+    setupTestServer();
+
     this.timeout(60000);
 
     let tmpDir = '/tmp/puppeteer-browsers-test';
@@ -56,6 +59,7 @@ describe('Chrome', () => {
         cacheDir: tmpDir,
         browser: Browser.CHROME,
         buildId: testChromeBuildId,
+        baseUrl: getServerUrl(),
       });
     });
 

packages/browsers/test/src/chromium/launcher.spec.ts

@@ -28,6 +28,7 @@ import {
   BrowserPlatform,
   Cache,
 } from '../../../lib/cjs/main.js';
+import {getServerUrl, setupTestServer} from '../utils.js';
 import {testChromiumBuildId} from '../versions.js';
 
 describe('Chromium', () => {
@@ -44,6 +45,8 @@ describe('Chromium', () => {
   });
 
   describe('launcher', function () {
+    setupTestServer();
+
     this.timeout(60000);
 
     let tmpDir = '/tmp/puppeteer-browsers-test';
@@ -56,6 +59,7 @@ describe('Chromium', () => {
         cacheDir: tmpDir,
         browser: Browser.CHROMIUM,
         buildId: testChromiumBuildId,
+        baseUrl: getServerUrl(),
       });
     });
 

packages/browsers/test/src/firefox/cli.spec.ts

@@ -20,12 +20,18 @@ import os from 'os';
 import path from 'path';
 
 import {CLI} from '../../../lib/cjs/CLI.js';
-import {createMockedReadlineInterface} from '../utils.js';
+import {
+  createMockedReadlineInterface,
+  getServerUrl,
+  setupTestServer,
+} from '../utils.js';
 import {testFirefoxBuildId} from '../versions.js';
 
 describe('Firefox CLI', function () {
   this.timeout(90000);
 
+  setupTestServer();
+
   let tmpDir = '/tmp/puppeteer-browsers-test';
 
   beforeEach(() => {
@@ -38,6 +44,7 @@ describe('Firefox CLI', function () {
       '@puppeteer/browsers',
       'clear',
       `--path=${tmpDir}`,
+      `--base-url=${getServerUrl()}`,
     ]);
   });
 
@@ -49,6 +56,7 @@ describe('Firefox CLI', function () {
       `firefox@${testFirefoxBuildId}`,
       `--path=${tmpDir}`,
       '--platform=linux',
+      `--base-url=${getServerUrl()}`,
     ]);
     assert.ok(
       fs.existsSync(
@@ -65,6 +73,7 @@ describe('Firefox CLI', function () {
       `firefox@latest`,
       `--path=${tmpDir}`,
       '--platform=linux',
+      `--base-url=${getServerUrl()}`,
     ]);
   });
 });

packages/browsers/test/src/firefox/fetch.spec.ts

@@ -20,6 +20,7 @@ import os from 'os';
 import path from 'path';
 
 import {fetch, Browser, BrowserPlatform, Cache} from '../../../lib/cjs/main.js';
+import {setupTestServer, getServerUrl} from '../utils.js';
 import {testFirefoxBuildId} from '../versions.js';
 
 /**
@@ -27,6 +28,8 @@ import {testFirefoxBuildId} from '../versions.js';
  * so it requires the network access.
  */
 describe('Firefox fetch', () => {
+  setupTestServer();
+
   let tmpDir = '/tmp/puppeteer-browsers-test';
 
   beforeEach(() => {
@@ -50,6 +53,7 @@ describe('Firefox fetch', () => {
       browser: Browser.FIREFOX,
       platform: BrowserPlatform.LINUX,
       buildId: testFirefoxBuildId,
+      baseUrl: getServerUrl(),
     });
     assert.strictEqual(browser.path, expectedOutputPath);
     assert.ok(fs.existsSync(expectedOutputPath));
@@ -60,7 +64,7 @@ describe('Firefox fetch', () => {
   (os.platform() === 'darwin' ? it : it.skip)(
     'should download a buildId that is a dmg archive',
     async function () {
-      this.timeout(120000);
+      this.timeout(180000);
       const expectedOutputPath = path.join(
         tmpDir,
         'firefox',
@@ -72,6 +76,7 @@ describe('Firefox fetch', () => {
         browser: Browser.FIREFOX,
         platform: BrowserPlatform.MAC,
         buildId: testFirefoxBuildId,
+        baseUrl: getServerUrl(),
       });
       assert.strictEqual(browser.path, expectedOutputPath);
       assert.ok(fs.existsSync(expectedOutputPath));

packages/browsers/test/src/firefox/launcher.spec.ts

@@ -29,6 +29,7 @@ import {
   Cache,
   createProfile,
 } from '../../../lib/cjs/main.js';
+import {setupTestServer, getServerUrl} from '../utils.js';
 import {testFirefoxBuildId} from '../versions.js';
 
 describe('Firefox', () => {
@@ -45,7 +46,9 @@ describe('Firefox', () => {
   });
 
   describe('launcher', function () {
-    this.timeout(60000);
+    this.timeout(120000);
+
+    setupTestServer();
 
     let tmpDir = '/tmp/puppeteer-browsers-test';
 
@@ -57,6 +60,7 @@ describe('Firefox', () => {
         cacheDir: tmpDir,
         browser: Browser.FIREFOX,
         buildId: testFirefoxBuildId,
+        baseUrl: getServerUrl(),
       });
     });
 

packages/browsers/test/src/utils.ts

@@ -14,9 +14,12 @@
  * limitations under the License.
  */
 
+import path from 'path';
 import * as readline from 'readline';
 import {Writable, Readable} from 'stream';
 
+import {TestServer} from '@pptr/testserver';
+
 export function createMockedReadlineInterface(
   input: string
 ): readline.Interface {
@@ -33,3 +36,29 @@ export function createMockedReadlineInterface(
     output: writable,
   });
 }
+
+const startServer = async () => {
+  const assetsPath = path.join(__dirname, '..', 'cache', 'server');
+  return await TestServer.create(assetsPath);
+};
+
+interface ServerState {
+  server: TestServer;
+}
+
+const state: Partial<ServerState> = {};
+
+export function setupTestServer(): void {
+  before(async () => {
+    state.server = await startServer();
+  });
+
+  after(async () => {
+    await state.server!.stop();
+    state.server = undefined;
+  });
+}
+
+export function getServerUrl(): string {
+  return `http://localhost:${state.server!.port}`;
+}

packages/browsers/tools/downloadTestBrowsers.mjs

@@ -0,0 +1,88 @@
+/**
+ * Copyright 2023 Google Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Downloads test browser binaries to test/cache/server folder that
+ * mirrors the structure of the download server.
+ */
+
+import {BrowserPlatform, fetch} from '@puppeteer/browsers';
+import path from 'path';
+import fs from 'fs';
+
+import * as versions from '../test/build/versions.js';
+import {downloadPaths} from '../lib/esm/browser-data/browser-data.js';
+
+function getBrowser(str) {
+  const regex = /test(.+)BuildId/;
+  const match = str.match(regex);
+
+  if (match && match[1]) {
+    return match[1].toLowerCase();
+  } else {
+    return null;
+  }
+}
+
+const cacheDir = path.normalize(path.join('.', 'test', 'cache'));
+const promises = [];
+
+for (const version of Object.keys(versions)) {
+  const browser = getBrowser(version);
+
+  if (!browser) {
+    continue;
+  }
+
+  const buildId = versions[version];
+
+  for (const platform of Object.values(BrowserPlatform)) {
+    promises.push(
+      (async function download(buildId, platform) {
+        const targetPath = path.join(
+          cacheDir,
+          'server',
+          ...downloadPaths[browser](platform, buildId)
+        );
+
+        if (fs.existsSync(targetPath)) {
+          return;
+        }
+
+        const result = await fetch({
+          browser,
+          buildId,
+          platform,
+          cacheDir: path.join(cacheDir, 'tmp'),
+          install: false,
+        });
+
+        fs.mkdirSync(path.dirname(targetPath), {
+          recursive: true,
+        });
+        fs.copyFileSync(result.path, targetPath);
+      })(buildId, platform)
+    );
+  }
+}
+
+await Promise.all(promises);
+
+fs.rmSync(path.join(cacheDir, 'tmp'), {
+  recursive: true,
+  force: true,
+  maxRetries: 10,
+});