feat!: use ~/.cache/puppeteer for browser downloads (#9095)

This commit is contained in:
jrandolf 2022-10-11 13:20:45 +02:00 committed by GitHub
parent 557d4a06c4
commit 3df375baed
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
28 changed files with 47 additions and 232 deletions

View File

@ -13,8 +13,6 @@ lib/
yarn.lock
.docusaurus/
.cache-loader
.local-chromium/
.local-firefox/
test/output-*/
.dev_profile*
coverage/

View File

@ -141,7 +141,7 @@ jobs:
- name: Setup cache for Chromium binary
uses: actions/cache@v3
with:
path: packages/puppeteer-core/.local-chromium
path: ~/.cache/puppeteer/chrome
key: ${{ runner.os }}-chromium-${{ hashFiles('packages/puppeteer-core/src/revisions.ts') }}
- name: Install Chromium
run: npm run postinstall
@ -184,7 +184,7 @@ jobs:
- name: Setup cache for Firefox binary
uses: actions/cache@v3
with:
path: packages/puppeteer-core/.local-firefox
path: ~/.cache/puppeteer/firefox
key: ${{ runner.os }}-firefox-${{ hashFiles('packages/puppeteer-core/src/revisions.ts') }}
- name: Install Firefox
env:

View File

@ -38,7 +38,7 @@ jobs:
# Ensure both a Chromium and a Firefox binary are available.
PUPPETEER_PRODUCT=firefox npm install
npm install
ls .local-chromium .local-firefox
ls ~/.cache/puppeteer
REV=$(node tools/check_availability.js -p linux)
echo "Installing revision $REV"
cat src/revisions.ts | sed "s/[0-9]\{6,\}/$REV/" > src/revisions.ts.replaced
@ -76,7 +76,7 @@ jobs:
# Ensure both a Chromium and a Firefox binary are available.
PUPPETEER_PRODUCT=firefox npm install
npm install
ls .local-chromium .local-firefox
ls ~/.cache/puppeteer
REV=$(node tools/check_availability.js -p linux)
echo "Installing revision $REV"
cat src/revisions.ts | sed "s/[0-9]\{6,\}/$REV/" > src/revisions.ts.replaced
@ -114,7 +114,7 @@ jobs:
# Ensure both a Chromium and a Firefox binary are available.
PUPPETEER_PRODUCT=firefox npm install
npm install
ls .local-chromium .local-firefox
ls ~/.cache/puppeteer
REV=$(node tools/check_availability.js -p linux)
echo "Installing revision $REV"
cat src/revisions.ts | sed "s/[0-9]\{6,\}/$REV/" > src/revisions.ts.replaced

2
.gitignore vendored
View File

@ -12,8 +12,6 @@ lib/
yarn.lock
.docusaurus/
.cache-loader
.local-chromium/
.local-firefox/
test/output-*/
.dev_profile*
coverage/

View File

@ -13,8 +13,6 @@ lib/
yarn.lock
.docusaurus/
.cache-loader
.local-chromium/
.local-firefox/
test/output-*/
.dev_profile*
coverage/

View File

@ -10,23 +10,18 @@ RUN apt-get update \
&& apt-get update \
&& apt-get install -y google-chrome-stable fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-khmeros fonts-kacst fonts-freefont-ttf libxss1 \
--no-install-recommends \
&& rm -rf /var/lib/apt/lists/*
&& rm -rf /var/lib/apt/lists/* \
&& groupadd -r pptruser && useradd -rm -g pptruser -G audio,video pptruser
USER pptruser
WORKDIR /home/pptruser
COPY puppeteer-latest.tgz /home/pptruser/puppeteer-latest.tgz
COPY puppeteer-core-latest.tgz /home/pptruser/puppeteer-core-latest.tgz
COPY puppeteer-latest.tgz puppeteer-core-latest.tgz ./
# Install puppeteer and puppeteer-core into /home/pptruser/node_modules.
RUN npm i ./puppeteer-core-latest.tgz ./puppeteer-latest.tgz \
&& rm ./puppeteer-core-latest.tgz ./puppeteer-latest.tgz \
# Add user so we don't need --no-sandbox.
# same layer as npm install to keep re-chowned files from using up several hundred MBs more space
&& groupadd -r pptruser && useradd -r -g pptruser -G audio,video pptruser \
&& mkdir -p /home/pptruser/Downloads \
&& chown -R pptruser:pptruser /home/pptruser \
&& (node -e "require('child_process').execSync(require('puppeteer').executablePath() + ' --credits', {stdio: 'inherit'})" > THIRD_PARTY_NOTICES)
USER pptruser
CMD ["google-chrome-stable"]

View File

@ -1,16 +0,0 @@
# Compatibility layer
This directory provides an additional compatibility layer between ES modules and CommonJS.
## Why?
Both `./cjs/compat.ts` and `./esm/compat.ts` are written as ES modules, but `./cjs/compat.ts` can additionally use NodeJS CommonJS globals such as `__dirname` and `require` while these are disabled in ES module mode. For more information, see [Differences between ES modules and CommonJS](https://nodejs.org/api/esm.html#differences-between-es-modules-and-commonjs).
## Adding exports
In order to add exports, two things need to be done:
- The exports must be declared in `src/compat.ts`.
- The exports must be realized in `./cjs/compat.ts` and `./esm/compat.ts`.
In the event `compat.ts` becomes too large, you can place declarations in another file. Just make sure `./cjs`, `./esm`, and `src` have the same structure.

View File

@ -1,19 +0,0 @@
import {dirname} from 'path';
/**
* @internal
*/
let puppeteerDirname: string;
try {
// In some environments, like esbuild, this will throw an error.
// We suppress the error since the bundled binary is not expected
// to be used or installed in this case and, therefore, the
// root directory does not have to be known.
puppeteerDirname = dirname(require.resolve('./compat'));
} catch (error) {
// Fallback to __dirname.
puppeteerDirname = __dirname;
}
export {puppeteerDirname};

View File

@ -1,7 +0,0 @@
{
"extends": "../../../../tsconfig.base.json",
"compilerOptions": {
"module": "CommonJS",
"outDir": "../../lib/cjs/puppeteer"
}
}

View File

@ -1,22 +0,0 @@
import {createRequire} from 'module';
import {dirname} from 'path';
import {fileURLToPath} from 'url';
const require = createRequire(import.meta.url);
/**
* @internal
*/
let puppeteerDirname: string;
try {
// In some environments, like esbuild, this will throw an error.
// We suppress the error since the bundled binary is not expected
// to be used or installed in this case and, therefore, the
// root directory does not have to be known.
puppeteerDirname = dirname(require.resolve('./compat'));
} catch (error) {
puppeteerDirname = dirname(fileURLToPath(import.meta.url));
}
export {puppeteerDirname};

View File

@ -1,6 +0,0 @@
{
"extends": "../../../../tsconfig.base.json",
"compilerOptions": {
"outDir": "../../lib/esm/puppeteer"
}
}

View File

@ -1,22 +0,0 @@
/**
* Copyright 2022 Google Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* @internal
*/
declare const puppeteerDirname: string;
export {puppeteerDirname};

View File

@ -14,10 +14,11 @@
* limitations under the License.
*/
import {dirname} from 'path';
import {puppeteerDirname} from './compat.js';
import {homedir} from 'os';
import {join} from 'path';
/**
* @internal
*/
export const rootDirname = dirname(dirname(dirname(puppeteerDirname)));
export const PUPPETEER_CACHE_DIR =
process.env['PUPPETEER_CACHE_DIR'] ?? join(homedir(), '.cache', 'puppeteer');

View File

@ -35,6 +35,7 @@ import * as util from 'util';
import {promisify} from 'util';
import {debug} from '../common/Debug.js';
import {Product} from '../common/Product.js';
import {PUPPETEER_CACHE_DIR} from '../constants.js';
import {assert} from '../util/assert.js';
const experimentalChromiumMacArm =
@ -62,11 +63,9 @@ const downloadURLs: Record<Product, Partial<Record<Platform, string>>> = {
const browserConfig = {
chrome: {
host: 'https://storage.googleapis.com',
destination: '.local-chromium',
},
firefox: {
host: 'https://archive.mozilla.org/pub/firefox/nightly/latest-mozilla-central',
destination: '.local-firefox',
},
} as const;
@ -159,10 +158,6 @@ export interface BrowserFetcherOptions {
* Determines the host that will be used for downloading.
*/
host?: string;
/**
* @internal
*/
projectRoot?: string;
}
/**
@ -213,7 +208,7 @@ export interface BrowserFetcherRevisionInfo {
export class BrowserFetcher {
#product: Product;
#downloadsFolder: string;
#downloadFolder: string;
#downloadHost: string;
#platform: Platform;
@ -227,9 +222,8 @@ export class BrowserFetcher {
`Unknown product: "${options.product}"`
);
this.#downloadsFolder =
options.path ||
path.join(options.projectRoot!, browserConfig[this.#product].destination);
this.#downloadFolder =
options.path || path.join(PUPPETEER_CACHE_DIR, this.#product);
this.#downloadHost = options.host || browserConfig[this.#product].host;
if (options.platform) {
@ -348,13 +342,13 @@ export class BrowserFetcher {
);
const fileName = url.split('/').pop();
assert(fileName, `A malformed download URL was found: ${url}.`);
const archivePath = path.join(this.#downloadsFolder, fileName);
const archivePath = path.join(this.#downloadFolder, fileName);
const outputPath = this.#getFolderPath(revision);
if (existsSync(outputPath)) {
return this.revisionInfo(revision);
}
if (!existsSync(this.#downloadsFolder)) {
await mkdir(this.#downloadsFolder);
if (!existsSync(this.#downloadFolder)) {
await mkdir(this.#downloadFolder, {recursive: true});
}
// Use system Chromium builds on Linux ARM devices
@ -384,10 +378,10 @@ export class BrowserFetcher {
* available locally on disk.
*/
async localRevisions(): Promise<string[]> {
if (!existsSync(this.#downloadsFolder)) {
if (!existsSync(this.#downloadFolder)) {
return [];
}
const fileNames = await readdir(this.#downloadsFolder);
const fileNames = await readdir(this.#downloadFolder);
return fileNames
.map(fileName => {
return parseFolderPath(this.#product, fileName);
@ -508,7 +502,7 @@ export class BrowserFetcher {
}
#getFolderPath(revision: string): string {
return path.resolve(this.#downloadsFolder, `${this.#platform}-${revision}`);
return path.resolve(this.#downloadFolder, `${this.#platform}-${revision}`);
}
}

View File

@ -20,10 +20,6 @@ import {tmpdir} from './util.js';
* @internal
*/
export class ChromeLauncher implements ProductLauncher {
/**
* @internal
*/
_projectRoot: string | undefined;
/**
* @internal
*/
@ -33,12 +29,7 @@ export class ChromeLauncher implements ProductLauncher {
*/
_isPuppeteerCore: boolean;
constructor(
projectRoot: string | undefined,
preferredRevision: string,
isPuppeteerCore: boolean
) {
this._projectRoot = projectRoot;
constructor(preferredRevision: string, isPuppeteerCore: boolean) {
this._preferredRevision = preferredRevision;
this._isPuppeteerCore = isPuppeteerCore;
}

View File

@ -19,10 +19,6 @@ import {tmpdir} from './util.js';
* @internal
*/
export class FirefoxLauncher implements ProductLauncher {
/**
* @internal
*/
_projectRoot: string | undefined;
/**
* @internal
*/
@ -32,12 +28,7 @@ export class FirefoxLauncher implements ProductLauncher {
*/
_isPuppeteerCore: boolean;
constructor(
projectRoot: string | undefined,
preferredRevision: string,
isPuppeteerCore: boolean
) {
this._projectRoot = projectRoot;
constructor(preferredRevision: string, isPuppeteerCore: boolean) {
this._preferredRevision = preferredRevision;
this._isPuppeteerCore = isPuppeteerCore;
}
@ -216,13 +207,7 @@ export class FirefoxLauncher implements ProductLauncher {
async _updateRevision(): Promise<void> {
// replace 'latest' placeholder with actual downloaded revision
if (this._preferredRevision === 'latest') {
if (!this._projectRoot) {
throw new Error(
'_projectRoot is undefined. Unable to create a BrowserFetcher.'
);
}
const browserFetcher = new BrowserFetcher({
projectRoot: this._projectRoot,
product: this.product,
});
const localRevisions = await browserFetcher.localRevisions();

View File

@ -125,8 +125,7 @@ export function resolveExecutablePath(
executablePath: string;
missingText?: string;
} {
const {product, _isPuppeteerCore, _projectRoot, _preferredRevision} =
launcher;
const {product, _isPuppeteerCore, _preferredRevision} = launcher;
let downloadPath: string | undefined;
// puppeteer-core doesn't take into account PUPPETEER_* env variables.
if (!_isPuppeteerCore) {
@ -155,13 +154,7 @@ export function resolveExecutablePath(
process.env['npm_config_puppeteer_download_path'] ||
process.env['npm_package_config_puppeteer_download_path'];
}
if (!_projectRoot) {
throw new Error(
'_projectRoot is undefined. Unable to create a BrowserFetcher.'
);
}
const browserFetcher = new BrowserFetcher({
projectRoot: _projectRoot,
product: product,
path: downloadPath,
});
@ -193,23 +186,14 @@ export function resolveExecutablePath(
* @internal
*/
export function createLauncher(
projectRoot: string | undefined,
preferredRevision: string,
isPuppeteerCore: boolean,
product: Product = 'chrome'
): ProductLauncher {
switch (product) {
case 'firefox':
return new FirefoxLauncher(
projectRoot,
preferredRevision,
isPuppeteerCore
);
return new FirefoxLauncher(preferredRevision, isPuppeteerCore);
case 'chrome':
return new ChromeLauncher(
projectRoot,
preferredRevision,
isPuppeteerCore
);
return new ChromeLauncher(preferredRevision, isPuppeteerCore);
}
}

View File

@ -75,7 +75,6 @@ export interface PuppeteerLaunchOptions
*/
export class PuppeteerNode extends Puppeteer {
#launcher?: ProductLauncher;
#projectRoot?: string;
#productName?: Product;
/**
@ -88,15 +87,12 @@ export class PuppeteerNode extends Puppeteer {
*/
constructor(
settings: {
projectRoot?: string;
preferredRevision?: string;
productName?: Product;
} & CommonPuppeteerSettings
) {
const {projectRoot, preferredRevision, productName, ...commonSettings} =
settings;
const {preferredRevision, productName, ...commonSettings} = settings;
super(commonSettings);
this.#projectRoot = projectRoot;
this.#productName = productName;
if (preferredRevision) {
this._preferredRevision = preferredRevision;
@ -203,7 +199,6 @@ export class PuppeteerNode extends Puppeteer {
}
this._changedProduct = false;
this.#launcher = createLauncher(
this.#projectRoot,
this._preferredRevision,
this._isPuppeteerCore,
this._productName
@ -241,6 +236,6 @@ export class PuppeteerNode extends Puppeteer {
* @returns A new BrowserFetcher instance.
*/
createBrowserFetcher(options: BrowserFetcherOptions): BrowserFetcher {
return new BrowserFetcher({...options, projectRoot: this.#projectRoot});
return new BrowserFetcher(options);
}
}

View File

@ -1,5 +1,3 @@
import {existsSync} from 'fs';
import {dirname, join, parse} from 'path';
import {tmpdir as osTmpDir} from 'os';
/**
@ -13,19 +11,3 @@ import {tmpdir as osTmpDir} from 'os';
export const tmpdir = (): string => {
return process.env['PUPPETEER_TMP_DIR'] || osTmpDir();
};
/**
* @internal
*/
export const getPackageDirectory = (from: string): string => {
let found = existsSync(join(from, 'package.json'));
const root = parse(from).root;
while (!found) {
if (from === root) {
throw new Error('Cannot find package directory');
}
from = dirname(from);
found = existsSync(join(from, 'package.json'));
}
return from;
};

View File

@ -19,22 +19,18 @@ export * from './common/Device.js';
export * from './common/Errors.js';
export * from './common/PredefinedNetworkConditions.js';
export * from './common/Puppeteer.js';
export * from './node/BrowserFetcher.js';
/**
* @deprecated Use the query handler API defined on {@link Puppeteer}
*/
export * from './common/QueryHandler.js';
export * from './node/BrowserFetcher.js';
import {rootDirname} from './constants.js';
import {PuppeteerNode} from './node/PuppeteerNode.js';
import {getPackageDirectory} from './node/util.js';
/**
* @public
*/
const puppeteer = new PuppeteerNode({
projectRoot: getPackageDirectory(rootDirname),
isPuppeteerCore: true,
});

View File

@ -4,8 +4,5 @@
"module": "CommonJS",
"outDir": "../lib/cjs/puppeteer"
},
"references": [
{"path": "../third_party/tsconfig.cjs.json"},
{"path": "../compat/cjs/tsconfig.json"}
]
"references": [{"path": "../third_party/tsconfig.cjs.json"}]
}

View File

@ -3,8 +3,5 @@
"compilerOptions": {
"outDir": "../lib/esm/puppeteer"
},
"references": [
{"path": "../third_party/tsconfig.json"},
{"path": "../compat/esm/tsconfig.json"}
]
"references": [{"path": "../third_party/tsconfig.json"}]
}

View File

@ -54,7 +54,6 @@ export * from './common/USKeyboardLayout.js';
export * from './common/util.js';
export * from './common/WaitTask.js';
export * from './common/WebWorker.js';
export * from './compat.d.js';
export * from './constants.js';
export * from './environment.js';
export * from './generated/injected.js';

View File

@ -18,6 +18,7 @@ import https, {RequestOptions} from 'https';
import createHttpsProxyAgent, {HttpsProxyAgentOptions} from 'https-proxy-agent';
import ProgressBar from 'progress';
import {getProxyForUrl} from 'proxy-from-env';
import {BrowserFetcher} from 'puppeteer-core';
import {PuppeteerNode} from 'puppeteer-core/internal/node/PuppeteerNode.js';
import {PUPPETEER_REVISIONS} from 'puppeteer-core/internal/revisions.js';
import URL from 'url';
@ -59,7 +60,7 @@ export async function downloadBrowser(): Promise<void> {
process.env['PUPPETEER_DOWNLOAD_PATH'] ||
process.env['npm_config_puppeteer_download_path'] ||
process.env['npm_package_config_puppeteer_download_path'];
const browserFetcher = puppeteer.createBrowserFetcher({
const browserFetcher = new BrowserFetcher({
product,
host: downloadHost,
path: downloadPath,

View File

@ -27,9 +27,7 @@ export * from 'puppeteer-core/internal/common/QueryHandler.js';
export {LaunchOptions} from 'puppeteer-core/internal/node/LaunchOptions.js';
import {Product} from 'puppeteer-core';
import {rootDirname} from 'puppeteer-core/internal/constants.js';
import {PuppeteerNode} from 'puppeteer-core/internal/node/PuppeteerNode.js';
import {getPackageDirectory} from 'puppeteer-core/internal/node/util.js';
import {PUPPETEER_REVISIONS} from 'puppeteer-core/internal/revisions.js';
const productName = (process.env['PUPPETEER_PRODUCT'] ||
@ -49,7 +47,6 @@ switch (productName) {
* @public
*/
const puppeteer = new PuppeteerNode({
projectRoot: getPackageDirectory(rootDirname),
preferredRevision,
isPuppeteerCore: false,
productName,

View File

@ -54,7 +54,6 @@ export * from 'puppeteer-core/internal/common/USKeyboardLayout.js';
export * from 'puppeteer-core/internal/common/util.js';
export * from 'puppeteer-core/internal/common/WaitTask.js';
export * from 'puppeteer-core/internal/common/WebWorker.js';
export * from 'puppeteer-core/internal/compat.d.js';
export * from 'puppeteer-core/internal/constants.js';
export * from 'puppeteer-core/internal/environment.js';
export * from 'puppeteer-core/internal/generated/injected.js';

View File

@ -18,15 +18,15 @@ import expect from 'expect';
import fs from 'fs';
import os from 'os';
import path from 'path';
import {BrowserFetcher, TimeoutError} from 'puppeteer';
import {Page} from 'puppeteer-core/internal/api/Page.js';
import {Product} from 'puppeteer-core/internal/common/Product.js';
import rimraf from 'rimraf';
import sinon from 'sinon';
import {TLSSocket} from 'tls';
import {promisify} from 'util';
import {Page} from 'puppeteer-core/internal/api/Page.js';
import {Product} from 'puppeteer-core/internal/common/Product.js';
import {getTestState, itOnlyRegularInstall} from './mocha-utils.js';
import utils from './utils.js';
import {TimeoutError} from 'puppeteer';
const mkdtempAsync = promisify(fs.mkdtemp);
const readFileAsync = promisify(fs.readFile);
@ -45,10 +45,10 @@ describe('Launcher specs', function () {
describe('Puppeteer', function () {
describe('BrowserFetcher', function () {
it('should download and extract chrome linux binary', async () => {
const {server, puppeteer} = getTestState();
const {server} = getTestState();
const downloadsFolder = await mkdtempAsync(TMP_FOLDER);
const browserFetcher = puppeteer.createBrowserFetcher({
const browserFetcher = new BrowserFetcher({
platform: 'linux',
path: downloadsFolder,
host: server.PREFIX,
@ -86,10 +86,10 @@ describe('Launcher specs', function () {
await rmAsync(downloadsFolder);
});
it('should download and extract firefox linux binary', async () => {
const {server, puppeteer} = getTestState();
const {server} = getTestState();
const downloadsFolder = await mkdtempAsync(TMP_FOLDER);
const browserFetcher = puppeteer.createBrowserFetcher({
const browserFetcher = new BrowserFetcher({
platform: 'linux',
path: downloadsFolder,
host: server.PREFIX,

View File

@ -71,7 +71,7 @@ TMPDIR="$(mktemp -d)"
cd $TMPDIR
npm install --loglevel silent $puppeteer_core_tarball $puppeteer_tarball
node --eval="require('puppeteer')"
ls $TMPDIR/node_modules/puppeteer-core/.local-chromium/
ls ~/.cache/puppeteer/chrome
echo "Testing... Chrome ES Modules"
TMPDIR="$(mktemp -d)"
@ -133,7 +133,7 @@ import puppeteer from 'puppeteer';
})();
" >>$TMPDIR/index.js
npx webpack
cp -r node_modules/puppeteer-core/.local-chromium .
cp -r ~/.cache/puppeteer/chrome .
rm -rf node_modules
node dist/bundle.cjs
@ -142,7 +142,7 @@ TMPDIR="$(mktemp -d)"
cd $TMPDIR
PUPPETEER_PRODUCT=firefox npm install --loglevel silent $puppeteer_core_tarball $puppeteer_tarball
node --eval="require('puppeteer')"
ls $TMPDIR/node_modules/puppeteer-core/.local-firefox
ls ~/.cache/puppeteer/firefox
echo "Testing... Firefox ES Modules"
TMPDIR="$(mktemp -d)"
@ -150,4 +150,4 @@ cd $TMPDIR
echo '{"type":"module"}' >>$TMPDIR/package.json
PUPPETEER_PRODUCT=firefox npm install --loglevel silent $puppeteer_core_tarball $puppeteer_tarball
node --input-type="module" --eval="import puppeteer from 'puppeteer'"
ls $TMPDIR/node_modules/puppeteer-core/.local-firefox
ls ~/.cache/puppeteer/firefox