From 3936600ba925f3e830c45c5d9354610590383eeb Mon Sep 17 00:00:00 2001 From: Alex Rudenko Date: Tue, 28 Mar 2023 11:27:50 +0200 Subject: [PATCH] refactor: use @puppeteer/browsers for install (#9898) Co-authored-by: Nikolay Vitkov <34244704+Lightning00Blade@users.noreply.github.com> --- .github/workflows/ci.yml | 4 +- docker/Dockerfile | 8 +- docker/pack.sh | 4 +- package-lock.json | 4 +- packages/puppeteer/package.json | 4 +- packages/puppeteer/src/node/install.ts | 236 ++++++------------ test/TestExpectations.json | 12 +- test/installation/package.json | 5 +- test/installation/src/constants.ts | 3 + test/installation/src/describeInstallation.ts | 5 +- .../src/puppeteer-configuration.spec.ts | 2 +- .../src/puppeteer-firefox.spec.ts | 2 +- .../src/puppeteer-webpack.spec.ts | 2 +- test/installation/src/puppeteer.spec.ts | 2 +- 14 files changed, 115 insertions(+), 178 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0953ae2bd23..a4dce6929ed 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -143,7 +143,7 @@ jobs: uses: actions/cache@v3 with: path: ~/.cache/puppeteer/chrome - key: ${{ runner.os }}-chromium-${{ hashFiles('packages/puppeteer-core/src/revisions.ts') }} + key: ${{ runner.os }}-chromium-${{ hashFiles('packages/puppeteer-core/src/revisions.ts') }}-${{ hashFiles('packages/puppeteer/src/node/install.ts') }} - name: Install Chromium run: npm run postinstall - name: Tests types @@ -211,7 +211,7 @@ jobs: uses: actions/cache@v3 with: path: ~/.cache/puppeteer/firefox - key: ${{ runner.os }}-firefox-${{ hashFiles('packages/puppeteer-core/src/revisions.ts') }} + key: ${{ runner.os }}-firefox-${{ hashFiles('packages/puppeteer-core/src/revisions.ts') }}-${{ hashFiles('packages/puppeteer/src/node/install.ts') }} - name: Install Firefox env: PUPPETEER_PRODUCT: firefox diff --git a/docker/Dockerfile b/docker/Dockerfile index 40d28cf0d46..16955fbca5a 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -17,11 +17,11 @@ USER pptruser WORKDIR /home/pptruser -COPY puppeteer-latest.tgz puppeteer-core-latest.tgz ./ +COPY puppeteer-browsers-latest.tgz puppeteer-latest.tgz puppeteer-core-latest.tgz ./ -# Install puppeteer and puppeteer-core into /home/pptruser/node_modules. -RUN npm i ./puppeteer-core-latest.tgz ./puppeteer-latest.tgz \ - && rm ./puppeteer-core-latest.tgz ./puppeteer-latest.tgz \ +# Install @puppeteer/browsers, puppeteer and puppeteer-core into /home/pptruser/node_modules. +RUN npm i ./puppeteer-browsers-latest.tgz ./puppeteer-core-latest.tgz ./puppeteer-latest.tgz \ + && rm ./puppeteer-browsers-latest.tgz ./puppeteer-core-latest.tgz ./puppeteer-latest.tgz \ && (node -e "require('child_process').execSync(require('puppeteer').executablePath() + ' --credits', {stdio: 'inherit'})" > THIRD_PARTY_NOTICES) CMD ["google-chrome-stable"] diff --git a/docker/pack.sh b/docker/pack.sh index c958c29a466..29229df9e78 100755 --- a/docker/pack.sh +++ b/docker/pack.sh @@ -7,10 +7,12 @@ set -e cd docker -npm pack --workspace puppeteer --workspace puppeteer-core --pack-destination . +npm pack --workspace puppeteer --workspace puppeteer-core --workspace @puppeteer/browsers --pack-destination . rm -f puppeteer-core-latest.tgz rm -f puppeteer-latest.tgz +rm -f puppeteer-browsers-latest.tgz mv puppeteer-core-*.tgz puppeteer-core-latest.tgz +mv puppeteer-browsers-*.tgz puppeteer-browsers-latest.tgz mv puppeteer-[0-9]*.tgz puppeteer-latest.tgz \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 48a5f3c0680..a79c1a980d7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9317,7 +9317,7 @@ }, "packages/browsers": { "name": "@puppeteer/browsers", - "version": "0.2.0", + "version": "0.3.0", "license": "Apache-2.0", "dependencies": { "debug": "4.3.4", @@ -9470,6 +9470,7 @@ "hasInstallScript": true, "license": "Apache-2.0", "dependencies": { + "@puppeteer/browsers": "0.3.0", "cosmiconfig": "8.1.3", "https-proxy-agent": "5.0.1", "progress": "2.0.3", @@ -14452,6 +14453,7 @@ "puppeteer": { "version": "file:packages/puppeteer", "requires": { + "@puppeteer/browsers": "0.3.0", "cosmiconfig": "8.1.3", "https-proxy-agent": "5.0.1", "progress": "2.0.3", diff --git a/packages/puppeteer/package.json b/packages/puppeteer/package.json index ba463f532de..6408d7a38ea 100644 --- a/packages/puppeteer/package.json +++ b/packages/puppeteer/package.json @@ -81,6 +81,7 @@ "clean": "if-file-deleted", "dependencies": [ "../puppeteer-core:build", + "../browsers:build", "generate:package-json" ], "files": [ @@ -119,6 +120,7 @@ "https-proxy-agent": "5.0.1", "progress": "2.0.3", "proxy-from-env": "1.1.0", - "puppeteer-core": "19.8.0" + "puppeteer-core": "19.8.0", + "@puppeteer/browsers": "0.3.0" } } diff --git a/packages/puppeteer/src/node/install.ts b/packages/puppeteer/src/node/install.ts index 6954fb85466..2ec3c7cebc5 100644 --- a/packages/puppeteer/src/node/install.ts +++ b/packages/puppeteer/src/node/install.ts @@ -14,13 +14,15 @@ * limitations under the License. */ -import https, {RequestOptions} from 'https'; -import URL from 'url'; - -import createHttpsProxyAgent, {HttpsProxyAgentOptions} from 'https-proxy-agent'; -import ProgressBar from 'progress'; -import {getProxyForUrl} from 'proxy-from-env'; -import {PuppeteerNode} from 'puppeteer-core/internal/node/PuppeteerNode.js'; +import { + fetch, + Browser, + resolveBuildId, + makeProgressCallback, + detectBrowserPlatform, + BrowserPlatform, +} from '@puppeteer/browsers'; +import {Product} from 'puppeteer-core'; import {PUPPETEER_REVISIONS} from 'puppeteer-core/internal/revisions.js'; import {getConfiguration} from '../getConfiguration.js'; @@ -29,6 +31,7 @@ import {getConfiguration} from '../getConfiguration.js'; * @internal */ const supportedProducts = { + chromium: 'Chromium', chrome: 'Chromium', firefox: 'Firefox Nightly', } as const; @@ -37,170 +40,69 @@ const supportedProducts = { * @internal */ export async function downloadBrowser(): Promise { + overrideProxy(); + const configuration = getConfiguration(); if (configuration.skipDownload) { logPolitely('**INFO** Skipping browser download as instructed.'); return; } - const puppeteer = new PuppeteerNode({configuration, isPuppeteerCore: false}); + let platform = detectBrowserPlatform(); + if (!platform) { + throw new Error('The current platform is not supported.'); + } + + // TODO: remove once Mac ARM is enabled by default for Puppeteer https://github.com/puppeteer/puppeteer/issues/9630. + if ( + platform === BrowserPlatform.MAC_ARM && + !configuration.experiments?.macArmChromiumEnabled + ) { + platform = BrowserPlatform.MAC; + } const product = configuration.defaultProduct!; - const browserFetcher = puppeteer.createBrowserFetcher(); + const browser = productToBrowser(product); - let revision = configuration.browserRevision; + // TODO: PUPPETEER_REVISIONS should use Chrome and not Chromium. + const unresolvedBuildId = + configuration.browserRevision || + PUPPETEER_REVISIONS[product === 'chrome' ? 'chromium' : 'firefox'] || + 'latest'; - if (!revision) { - switch (product) { - case 'chrome': - revision = PUPPETEER_REVISIONS.chromium; - break; - case 'firefox': - revision = PUPPETEER_REVISIONS.firefox; - revision = await getFirefoxNightlyVersion(); - break; - } - } + const buildId = await resolveBuildId(browser, platform, unresolvedBuildId); - await fetchBinary(revision); - - function fetchBinary(revision: string) { - const revisionInfo = browserFetcher.revisionInfo(revision); - - // Do nothing if the revision is already downloaded. - if (revisionInfo.local) { - logPolitely( - `${supportedProducts[product]} is already in ${revisionInfo.folderPath}; skipping download.` - ); - return; - } - - // Override current environment proxy settings with npm configuration, if any. - const NPM_HTTPS_PROXY = - process.env['npm_config_https_proxy'] || process.env['npm_config_proxy']; - const NPM_HTTP_PROXY = - process.env['npm_config_http_proxy'] || process.env['npm_config_proxy']; - const NPM_NO_PROXY = process.env['npm_config_no_proxy']; - - if (NPM_HTTPS_PROXY) { - process.env['HTTPS_PROXY'] = NPM_HTTPS_PROXY; - } - if (NPM_HTTP_PROXY) { - process.env['HTTP_PROXY'] = NPM_HTTP_PROXY; - } - if (NPM_NO_PROXY) { - process.env['NO_PROXY'] = NPM_NO_PROXY; - } - - function onSuccess(localRevisions: string[]): void { - logPolitely( - `${supportedProducts[product]} (${revisionInfo.revision}) downloaded to ${revisionInfo.folderPath}` - ); - const otherRevisions = localRevisions.filter(revision => { - return revision !== revisionInfo.revision; - }); - if (otherRevisions.length) { - logPolitely( - `Other installed ${ - supportedProducts[product] - } browsers in ${browserFetcher.getDownloadPath()} include: ${otherRevisions.join( - ', ' - )}. Remove old revisions from ${browserFetcher.getDownloadPath()} if you don't need them.` - ); - } - } - - function onError(error: Error) { - console.error( - `ERROR: Failed to set up ${supportedProducts[product]} r${revision}! Set "PUPPETEER_SKIP_DOWNLOAD" env variable to skip download.` - ); - console.error(error); - process.exit(1); - } - - let progressBar: ProgressBar | null = null; - let lastDownloadedBytes = 0; - function onProgress(downloadedBytes: number, totalBytes: number) { - if (!progressBar) { - progressBar = new ProgressBar( - `Downloading ${ - supportedProducts[product] - } r${revision} - ${toMegabytes(totalBytes)} [:bar] :percent :etas `, - { - complete: '=', - incomplete: ' ', - width: 20, - total: totalBytes, - } - ); - } - const delta = downloadedBytes - lastDownloadedBytes; - lastDownloadedBytes = downloadedBytes; - progressBar.tick(delta); - } - - return browserFetcher - .download(revisionInfo.revision, onProgress) - .then(() => { - return browserFetcher.localRevisions(); - }) - .then(onSuccess) - .catch(onError); - } - - function toMegabytes(bytes: number) { - const mb = bytes / 1024 / 1024; - return `${Math.round(mb * 10) / 10} Mb`; - } - - async function getFirefoxNightlyVersion(): Promise { - const firefoxVersionsUrl = - 'https://product-details.mozilla.org/1.0/firefox_versions.json'; - - const proxyURL = getProxyForUrl(firefoxVersionsUrl); - - const requestOptions: RequestOptions = {}; - - if (proxyURL) { - const parsedProxyURL = URL.parse(proxyURL); - - const proxyOptions = { - ...parsedProxyURL, - secureProxy: parsedProxyURL.protocol === 'https:', - } as HttpsProxyAgentOptions; - - requestOptions.agent = createHttpsProxyAgent(proxyOptions); - requestOptions.rejectUnauthorized = false; - } - - const promise = new Promise((resolve, reject) => { - let data = ''; - logPolitely( - `Requesting latest Firefox Nightly version from ${firefoxVersionsUrl}` - ); - https - .get(firefoxVersionsUrl, requestOptions, r => { - if (r.statusCode && r.statusCode >= 400) { - return reject(new Error(`Got status code ${r.statusCode}`)); - } - r.on('data', chunk => { - data += chunk; - }); - r.on('end', () => { - try { - const versions = JSON.parse(data); - return resolve(versions.FIREFOX_NIGHTLY); - } catch { - return reject(new Error('Firefox version not found')); - } - }); - }) - .on('error', reject); + try { + const result = await fetch({ + browser, + cacheDir: configuration.cacheDirectory!, + platform, + buildId, + downloadProgressCallback: makeProgressCallback(browser, buildId), }); - return promise; + + logPolitely( + `${supportedProducts[product]} (${result.buildId}) downloaded to ${result.path}` + ); + } catch (error) { + console.error( + `ERROR: Failed to set up ${supportedProducts[product]} r${buildId}! Set "PUPPETEER_SKIP_DOWNLOAD" env variable to skip download.` + ); + console.error(error); + process.exit(1); } } +function productToBrowser(product?: Product) { + switch (product) { + case 'chrome': + return Browser.CHROMIUM; + case 'firefox': + return Browser.FIREFOX; + } + return Browser.CHROMIUM; +} + /** * @internal */ @@ -213,3 +115,25 @@ function logPolitely(toBeLogged: unknown): void { console.log(toBeLogged); } } + +/** + * @internal + */ +function overrideProxy() { + // Override current environment proxy settings with npm configuration, if any. + const NPM_HTTPS_PROXY = + process.env['npm_config_https_proxy'] || process.env['npm_config_proxy']; + const NPM_HTTP_PROXY = + process.env['npm_config_http_proxy'] || process.env['npm_config_proxy']; + const NPM_NO_PROXY = process.env['npm_config_no_proxy']; + + if (NPM_HTTPS_PROXY) { + process.env['HTTPS_PROXY'] = NPM_HTTPS_PROXY; + } + if (NPM_HTTP_PROXY) { + process.env['HTTP_PROXY'] = NPM_HTTP_PROXY; + } + if (NPM_NO_PROXY) { + process.env['NO_PROXY'] = NPM_NO_PROXY; + } +} diff --git a/test/TestExpectations.json b/test/TestExpectations.json index e163144a9dc..94b41ae8078 100644 --- a/test/TestExpectations.json +++ b/test/TestExpectations.json @@ -305,12 +305,6 @@ "parameters": ["webDriverBiDi"], "expectations": ["FAIL", "TIMEOUT"] }, - { - "testIdPattern": "[navigation.spec] navigation Page.goto should not leak listeners during navigation of 11 pages", - "platforms": ["darwin"], - "parameters": ["chrome", "webDriverBiDi"], - "expectations": ["PASS", "TIMEOUT"] - }, { "testIdPattern": "[navigation.spec] navigation Page.goto should not throw an error for a 404 response with an empty body", "platforms": ["darwin", "linux", "win32"], @@ -1175,6 +1169,12 @@ "parameters": ["chrome", "webDriverBiDi"], "expectations": ["FAIL", "PASS", "TIMEOUT"] }, + { + "testIdPattern": "[navigation.spec] navigation Page.goto should not leak listeners during navigation of 11 pages", + "platforms": ["darwin"], + "parameters": ["chrome", "webDriverBiDi"], + "expectations": ["PASS", "TIMEOUT"] + }, { "testIdPattern": "[navigation.spec] navigation Page.goto should send referer", "platforms": ["darwin", "linux", "win32"], diff --git a/test/installation/package.json b/test/installation/package.json index e60bb8a46cf..9b11b9f8d00 100644 --- a/test/installation/package.json +++ b/test/installation/package.json @@ -26,10 +26,11 @@ ] }, "build:packages": { - "command": "npm pack --workspace puppeteer --workspace puppeteer-core", + "command": "npm pack --workspace puppeteer --workspace puppeteer-core --workspace @puppeteer/browsers", "dependencies": [ "../../packages/puppeteer:build", - "../../packages/puppeteer-core:build" + "../../packages/puppeteer-core:build", + "../../packages/browsers:build" ], "files": [], "output": [ diff --git a/test/installation/src/constants.ts b/test/installation/src/constants.ts index e41c98ea00b..e78c93b159e 100644 --- a/test/installation/src/constants.ts +++ b/test/installation/src/constants.ts @@ -22,6 +22,9 @@ import glob from 'glob'; export const PUPPETEER_CORE_PACKAGE_PATH = resolve( glob.sync('puppeteer-core-*.tgz')[0]! ); +export const PUPPETEER_BROWSERS_PACKAGE_PATH = resolve( + glob.sync('puppeteer-browsers-[0-9]*.tgz')[0]! +); export const PUPPETEER_PACKAGE_PATH = resolve( glob.sync('puppeteer-[0-9]*.tgz')[0]! ); diff --git a/test/installation/src/describeInstallation.ts b/test/installation/src/describeInstallation.ts index 706f81a339b..61defefb93e 100644 --- a/test/installation/src/describeInstallation.ts +++ b/test/installation/src/describeInstallation.ts @@ -22,6 +22,7 @@ import {join} from 'path'; import { PUPPETEER_CORE_PACKAGE_PATH, PUPPETEER_PACKAGE_PATH, + PUPPETEER_BROWSERS_PACKAGE_PATH, } from './constants.js'; import {execFile} from './util.js'; @@ -78,6 +79,8 @@ export const describeInstallation = ( return PUPPETEER_PACKAGE_PATH; case 'puppeteer-core': return PUPPETEER_CORE_PACKAGE_PATH; + case '@puppeteer/browsers': + return PUPPETEER_BROWSERS_PACKAGE_PATH; default: return module; } @@ -102,7 +105,7 @@ export const describeInstallation = ( after(async () => { if (process.env['KEEP_SANDBOX']) { - await rm(sandbox, {recursive: true, force: true}); + await rm(sandbox, {recursive: true, force: true, maxRetries: 5}); } }); diff --git a/test/installation/src/puppeteer-configuration.spec.ts b/test/installation/src/puppeteer-configuration.spec.ts index b4daf6be563..bf227b628ea 100644 --- a/test/installation/src/puppeteer-configuration.spec.ts +++ b/test/installation/src/puppeteer-configuration.spec.ts @@ -24,7 +24,7 @@ import {readAsset} from './util.js'; describeInstallation( '`puppeteer` with configuration', { - dependencies: ['puppeteer-core', 'puppeteer'], + dependencies: ['puppeteer-core', '@puppeteer/browsers', 'puppeteer'], before: async cwd => { await writeFile( join(cwd, '.puppeteerrc.cjs'), diff --git a/test/installation/src/puppeteer-firefox.spec.ts b/test/installation/src/puppeteer-firefox.spec.ts index debd2d60ccb..bf3105396ad 100644 --- a/test/installation/src/puppeteer-firefox.spec.ts +++ b/test/installation/src/puppeteer-firefox.spec.ts @@ -24,7 +24,7 @@ import {readAsset} from './util.js'; describeInstallation( '`puppeteer` with Firefox', { - dependencies: ['puppeteer-core', 'puppeteer'], + dependencies: ['@puppeteer/browsers', 'puppeteer-core', 'puppeteer'], env: cwd => { return { PUPPETEER_CACHE_DIR: join(cwd, '.cache', 'puppeteer'), diff --git a/test/installation/src/puppeteer-webpack.spec.ts b/test/installation/src/puppeteer-webpack.spec.ts index 63482721027..75b6a91c964 100644 --- a/test/installation/src/puppeteer-webpack.spec.ts +++ b/test/installation/src/puppeteer-webpack.spec.ts @@ -23,7 +23,7 @@ import {execFile, readAsset} from './util.js'; describeInstallation( '`puppeteer` with Webpack', { - dependencies: ['puppeteer-core', 'puppeteer'], + dependencies: ['@puppeteer/browsers', 'puppeteer-core', 'puppeteer'], devDependencies: ['webpack', 'webpack-cli'], env: cwd => { return { diff --git a/test/installation/src/puppeteer.spec.ts b/test/installation/src/puppeteer.spec.ts index 22585d09116..2dbf384409c 100644 --- a/test/installation/src/puppeteer.spec.ts +++ b/test/installation/src/puppeteer.spec.ts @@ -24,7 +24,7 @@ import {readAsset} from './util.js'; describeInstallation( '`puppeteer`', { - dependencies: ['puppeteer-core', 'puppeteer'], + dependencies: ['@puppeteer/browsers', 'puppeteer-core', 'puppeteer'], env: cwd => { return { PUPPETEER_CACHE_DIR: join(cwd, '.cache', 'puppeteer'),