From 6841bd68d85e3b3952c5e7ce454ac4d23f84262d Mon Sep 17 00:00:00 2001 From: jrandolf <101637635+jrandolf@users.noreply.github.com> Date: Mon, 9 May 2022 13:17:24 +0200 Subject: [PATCH] feat: support ES modules (#8306) --- .versionrc.js | 3 +- compat/README.md | 16 +++++ compat/cjs/compat.ts | 16 +++++ compat/cjs/tsconfig.json | 9 +++ compat/esm/compat.ts | 19 ++++++ compat/esm/tsconfig.json | 9 +++ package.json | 23 +++++-- scripts/test-install.sh | 98 +++++++++++++++++++++++++----- src/common/Debug.ts | 5 +- src/compat.ts | 3 + src/constants.ts | 4 ++ src/generated/README.md | 3 + src/generated/version.ts | 1 + src/initialize-node.ts | 17 +----- src/node/BrowserFetcher.ts | 7 +-- src/node/NodeWebSocketTransport.ts | 7 +-- src/templates/version.ts.tmpl | 1 + utils/apply_next_version.js | 1 - utils/generate_version_file.js | 9 +++ utils/prepare_puppeteer_core.js | 6 +- 20 files changed, 211 insertions(+), 46 deletions(-) create mode 100644 compat/README.md create mode 100644 compat/cjs/compat.ts create mode 100644 compat/cjs/tsconfig.json create mode 100644 compat/esm/compat.ts create mode 100644 compat/esm/tsconfig.json create mode 100644 src/compat.ts create mode 100644 src/constants.ts create mode 100644 src/generated/README.md create mode 100644 src/generated/version.ts create mode 100644 src/templates/version.ts.tmpl create mode 100644 utils/generate_version_file.js diff --git a/.versionrc.js b/.versionrc.js index 117096cbc7e..f8739d98308 100644 --- a/.versionrc.js +++ b/.versionrc.js @@ -20,7 +20,8 @@ module.exports = { tag: true, }, scripts: { - prerelease: 'node utils/remove_version_suffix.js', + prerelease: + 'node utils/remove_version_suffix.js && node utils/generate_version_file.js', postbump: 'IS_RELEASE=true npm run doc && git add --update', }, }; diff --git a/compat/README.md b/compat/README.md new file mode 100644 index 00000000000..a72ecab4e8a --- /dev/null +++ b/compat/README.md @@ -0,0 +1,16 @@ +# Compatibility layer + +This directory provides an additional compatibility layer between ES modules and CommonJS. + +## Why? + +Both `./cjs/compat.ts` and `./esm/compat.ts` are written as ES modules, but `./cjs/compat.ts` can additionally use NodeJS CommonJS globals such as `__dirname` and `require` while these are disabled in ES module mode. For more information, see [Differences between ES modules and CommonJS](https://nodejs.org/api/esm.html#differences-between-es-modules-and-commonjs). + +## Adding exports + +In order to add exports, two things need to be done: + +- The exports must be declared in `src/compat.ts`. +- The exports must be realized in `./cjs/compat.ts` and `./esm/compat.ts`. + +In the event `compat.ts` becomes too large, you can place declarations in another file. Just make sure `./cjs`, `./esm`, and `src` have the same structure. diff --git a/compat/cjs/compat.ts b/compat/cjs/compat.ts new file mode 100644 index 00000000000..6efa3fe9aab --- /dev/null +++ b/compat/cjs/compat.ts @@ -0,0 +1,16 @@ +import { dirname } from 'path'; + +let puppeteerDirname: string; + +try { + // In some environments, like esbuild, this will throw an error. + // We suppress the error since the bundled binary is not expected + // to be used or installed in this case and, therefore, the + // root directory does not have to be known. + puppeteerDirname = dirname(require.resolve('./compat')); +} catch (error) { + // Fallback to __dirname. + puppeteerDirname = __dirname; +} + +export { puppeteerDirname }; diff --git a/compat/cjs/tsconfig.json b/compat/cjs/tsconfig.json new file mode 100644 index 00000000000..2bcb2b984b4 --- /dev/null +++ b/compat/cjs/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "composite": true, + "outDir": "../../lib/cjs/puppeteer", + "module": "CommonJS" + }, + "references": [{ "path": "../../vendor/tsconfig.cjs.json" }] +} diff --git a/compat/esm/compat.ts b/compat/esm/compat.ts new file mode 100644 index 00000000000..a3288e58535 --- /dev/null +++ b/compat/esm/compat.ts @@ -0,0 +1,19 @@ +import { createRequire } from 'module'; +import { dirname } from 'path'; +import { fileURLToPath } from 'url'; + +const require = createRequire(import.meta.url); + +let puppeteerDirname: string; + +try { + // In some environments, like esbuild, this will throw an error. + // We suppress the error since the bundled binary is not expected + // to be used or installed in this case and, therefore, the + // root directory does not have to be known. + puppeteerDirname = dirname(require.resolve('./compat')); +} catch (error) { + puppeteerDirname = dirname(fileURLToPath(import.meta.url)); +} + +export { puppeteerDirname }; diff --git a/compat/esm/tsconfig.json b/compat/esm/tsconfig.json new file mode 100644 index 00000000000..42b320fcd3f --- /dev/null +++ b/compat/esm/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "composite": true, + "outDir": "../../lib/esm/puppeteer", + "module": "esnext" + }, + "references": [{ "path": "../../vendor/tsconfig.esm.json" }] +} diff --git a/package.json b/package.json index f4e478051d3..16ed6235fa1 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,18 @@ "headless", "automation" ], + "type": "commonjs", "main": "./cjs-entry.js", + "exports": { + ".": { + "import": "./lib/esm/puppeteer/node.js", + "require": "./cjs-entry.js" + }, + "./*": { + "import": "./*", + "require": "./*" + } + }, "types": "lib/types.d.ts", "repository": "github:puppeteer/puppeteer", "engines": { @@ -23,10 +34,10 @@ "unit-with-coverage": "cross-env COVERAGE=1 npm run unit", "assert-unit-coverage": "cross-env COVERAGE=1 mocha --config mocha-config/coverage-tests.js", "funit": "cross-env PUPPETEER_PRODUCT=firefox npm run unit", - "test": "npm run tsc && npm run lint --silent && npm run unit-with-coverage && npm run test-browser", + "test": "npm run build && npm run lint --silent && npm run unit-with-coverage && npm run test-browser", "prepare": "node typescript-if-required.js && ([[ $HUSKY = 0 ]] || husky install)", "prepublishOnly": "npm run build", - "dev-install": "npm run tsc && node install.js", + "dev-install": "npm run build && node install.js", "install": "node install.js", "eslint": "([ \"$CI\" = true ] && eslint --ext js --ext ts --quiet -f codeframe . || eslint --ext js --ext ts .)", "eslint-fix": "eslint --ext js --ext ts --fix .", @@ -37,15 +48,18 @@ "doc": "node utils/doclint/cli.js", "generate-api-docs-for-testing": "commonmark docs/api.md > docs/api.html", "clean-lib": "rimraf lib", - "build": "npm run tsc && npm run generate-d-ts", - "tsc": "npm run clean-lib && tsc --version && npm run tsc-cjs && npm run tsc-esm", + "build": "npm run tsc && npm run generate-d-ts && npm run generate-esm-package-json", + "tsc": "npm run clean-lib && tsc --version && (npm run tsc-cjs & npm run tsc-esm) && (npm run tsc-compat-cjs & npm run tsc-compat-esm)", "tsc-cjs": "tsc -b src/tsconfig.cjs.json", "tsc-esm": "tsc -b src/tsconfig.esm.json", + "tsc-compat-cjs": "tsc -b compat/cjs/tsconfig.json", + "tsc-compat-esm": "tsc -b compat/esm/tsconfig.json", "apply-next-version": "node utils/apply_next_version.js", "test-install": "scripts/test-install.sh", "clean-docs": "rimraf website/docs && rimraf docs-api-json", "generate-d-ts": "npm run clean-docs && api-extractor run --local --verbose", "generate-docs": "npm run generate-d-ts && api-documenter markdown -i docs-api-json -o website/docs && node utils/remove-tag.js", + "generate-esm-package-json": "echo '{\"type\": \"module\"}' > lib/esm/package.json", "ensure-correct-devtools-protocol-revision": "ts-node -s scripts/ensure-correct-devtools-protocol-package", "ensure-pinned-deps": "ts-node -s scripts/ensure-pinned-deps", "test-types-file": "ts-node -s scripts/test-ts-definition-files.ts", @@ -58,6 +72,7 @@ "lib/**/*.d.ts.map", "lib/**/*.js", "lib/**/*.js.map", + "lib/**/package.json", "install.js", "typescript-if-required.js", "cjs-entry.js", diff --git a/scripts/test-install.sh b/scripts/test-install.sh index e59aa39d00b..44ab93a930e 100755 --- a/scripts/test-install.sh +++ b/scripts/test-install.sh @@ -1,41 +1,111 @@ #!/usr/bin/env sh set -e +# All tests are headed by a echo 'Test'. +# The general schema is: +# 1. Check we can install from the tarball. +# 2. The install script works and correctly exits without errors +# 3. Requiring/importing Puppeteer from Node works. + +## Puppeter tests + +echo "Setting up Puppeteer tests..." ROOTDIR="$(pwd)" -# Pack the module into a tarball npm pack tarball="$(realpath puppeteer-*.tgz)" + +echo "Testing... Chrome CommonJS" TMPDIR="$(mktemp -d)" cd $TMPDIR -# Check we can install from the tarball. -# This emulates installing from npm and ensures that: -# 1. we publish the right files in the `files` list from package.json -# 2. The install script works and correctly exits without errors -# 3. Requiring Puppeteer from Node works. npm install --loglevel silent "${tarball}" node --eval="require('puppeteer')" +node --eval="require('puppeteer/lib/cjs/puppeteer/revisions.js');" ls $TMPDIR/node_modules/puppeteer/.local-chromium/ -# Again for Firefox +echo "Testing... Chrome ES Modules" +TMPDIR="$(mktemp -d)" +cd $TMPDIR +echo '{"type":"module"}' >>$TMPDIR/package.json +npm install --loglevel silent "${tarball}" +node --input-type="module" --eval="import puppeteer from 'puppeteer'" +node --input-type="module" --eval="import 'puppeteer/lib/esm/puppeteer/revisions.js';" +node --input-type="module" --eval=" +import puppeteer from 'puppeteer'; +(async () => { + const browser = await puppeteer.launch(); + const page = await browser.newPage(); + await page.goto('http://example.com'); + await page.screenshot({ path: 'example.png' }); + await browser.close(); +})(); +" + +echo "Testing... Chrome Webpack ES Modules" +TMPDIR="$(mktemp -d)" +cd $TMPDIR +echo '{"type": "module"}' >>$TMPDIR/package.json +npm install --loglevel silent "${tarball}" +npm install -D --loglevel silent webpack webpack-cli@4.9.2 +echo 'export default { + mode: "production", + entry: "./index.js", + target: "node", + output: { + filename: "bundle.cjs", + }, +};' >>$TMPDIR/webpack.config.js +echo " +import puppeteer from 'puppeteer'; +(async () => { + const browser = await puppeteer.launch(); + const page = await browser.newPage(); + await page.goto('http://example.com'); + await page.screenshot({ path: 'example.png' }); + await browser.close(); +})(); +" >>$TMPDIR/index.js +npx webpack +cp -r node_modules/puppeteer/.local-chromium . +rm -rf node_modules +node dist/bundle.cjs + +echo "Testing... Firefox CommonJS" TMPDIR="$(mktemp -d)" cd $TMPDIR PUPPETEER_PRODUCT=firefox npm install --loglevel silent "${tarball}" node --eval="require('puppeteer')" -rm "${tarball}" +node --eval="require('puppeteer/lib/cjs/puppeteer/revisions.js');" ls $TMPDIR/node_modules/puppeteer/.local-firefox/linux-*/firefox/firefox -# Again for puppeteer-core +echo "Testing... Firefox ES Modules" +TMPDIR="$(mktemp -d)" +cd $TMPDIR +echo '{"type":"module"}' >>$TMPDIR/package.json +PUPPETEER_PRODUCT=firefox npm install --loglevel silent "${tarball}" +node --input-type="module" --eval="import puppeteer from 'puppeteer'" +node --input-type="module" --eval="import 'puppeteer/lib/esm/puppeteer/revisions.js';" +ls $TMPDIR/node_modules/puppeteer/.local-firefox/linux-*/firefox/firefox + +## Puppeteer Core tests + +echo "Setting up Puppeteer Core tests..." cd $ROOTDIR +rm "${tarball}" node ./utils/prepare_puppeteer_core.js npm pack tarball="$(realpath puppeteer-core-*.tgz)" + +echo "Testing... Puppeteer Core CommonJS" TMPDIR="$(mktemp -d)" cd $TMPDIR -# Check we can install from the tarball. -# This emulates installing from npm and ensures that: -# 1. we publish the right files in the `files` list from package.json -# 2. The install script works and correctly exits without errors -# 3. Requiring Puppeteer Core from Node works. npm install --loglevel silent "${tarball}" node --eval="require('puppeteer-core')" +node --eval="require('puppeteer-core/lib/cjs/puppeteer/revisions.js');" +echo "Testing... Puppeteer Core ES Modules" +TMPDIR="$(mktemp -d)" +cd $TMPDIR +echo '{"type":"module"}' >>$TMPDIR/package.json +npm install --loglevel silent "${tarball}" +node --input-type="module" --eval="import puppeteer from 'puppeteer-core'" +node --input-type="module" --eval="import 'puppeteer-core/lib/esm/puppeteer/revisions.js';" diff --git a/src/common/Debug.ts b/src/common/Debug.ts index 8ff124b094b..10531987046 100644 --- a/src/common/Debug.ts +++ b/src/common/Debug.ts @@ -54,8 +54,9 @@ import { isNode } from '../environment.js'; */ export const debug = (prefix: string): ((...args: unknown[]) => void) => { if (isNode) { - // eslint-disable-next-line @typescript-eslint/no-var-requires - return require('debug')(prefix); + return async (...logArgs: unknown[]) => { + (await import('debug')).default(prefix)(logArgs); + }; } return (...logArgs: unknown[]): void => { diff --git a/src/compat.ts b/src/compat.ts new file mode 100644 index 00000000000..873dcea7966 --- /dev/null +++ b/src/compat.ts @@ -0,0 +1,3 @@ +declare const puppeteerDirname: string; + +export { puppeteerDirname }; diff --git a/src/constants.ts b/src/constants.ts new file mode 100644 index 00000000000..71ef76103c8 --- /dev/null +++ b/src/constants.ts @@ -0,0 +1,4 @@ +import { dirname } from 'path'; +import { puppeteerDirname } from './compat.js'; + +export const rootDirname = dirname(dirname(dirname(puppeteerDirname))); diff --git a/src/generated/README.md b/src/generated/README.md new file mode 100644 index 00000000000..3e5c7eb037f --- /dev/null +++ b/src/generated/README.md @@ -0,0 +1,3 @@ +# Generated Artifacts + +**Do not edit manually edit any TypeScript files in this folder** All TS files are generated from their respectively named template file (ext. `tmpl`) in the `templates` directory. Edit them there is needed. diff --git a/src/generated/version.ts b/src/generated/version.ts new file mode 100644 index 00000000000..b7026ce6bba --- /dev/null +++ b/src/generated/version.ts @@ -0,0 +1 @@ +export const packageVersion = '13.7.0'; diff --git a/src/initialize-node.ts b/src/initialize-node.ts index 3d38a47069d..9b55b98975f 100644 --- a/src/initialize-node.ts +++ b/src/initialize-node.ts @@ -17,24 +17,11 @@ import { PuppeteerNode } from './node/Puppeteer.js'; import { PUPPETEER_REVISIONS } from './revisions.js'; import { sync } from 'pkg-dir'; -import { dirname } from 'path'; import { Product } from './common/Product.js'; - -function resolvePuppeteerRootDirectory(): string | undefined { - try { - // In some environments, like esbuild, this will throw an error. - // We suppress the error since the bundled binary is not expected - // to be used or installed in this case and, therefore, the - // root directory does not have to be known. - return sync(dirname(require.resolve('./initialize-node'))); - } catch (error) { - // Fallback to __dirname. - return sync(__dirname); - } -} +import { rootDirname } from './constants.js'; export const initializePuppeteerNode = (packageName: string): PuppeteerNode => { - const puppeteerRootDirectory = resolvePuppeteerRootDirectory(); + const puppeteerRootDirectory = sync(rootDirname); let preferredRevision = PUPPETEER_REVISIONS.chromium; const isPuppeteerCore = packageName === 'puppeteer-core'; // puppeteer-core ignores environment variables diff --git a/src/node/BrowserFetcher.ts b/src/node/BrowserFetcher.ts index 2323860ecae..fdd89da086d 100644 --- a/src/node/BrowserFetcher.ts +++ b/src/node/BrowserFetcher.ts @@ -35,6 +35,9 @@ import createHttpsProxyAgent, { import { getProxyForUrl } from 'proxy-from-env'; import { assert } from '../common/assert.js'; +import tar from 'tar-fs'; +import bzip from 'unbzip2-stream'; + const { PUPPETEER_EXPERIMENTAL_CHROMIUM_MAC_ARM } = process.env; const debugFetcher = debug('puppeteer:fetcher'); @@ -512,10 +515,6 @@ function install(archivePath: string, folderPath: string): Promise { * @internal */ function extractTar(tarPath: string, folderPath: string): Promise { - // eslint-disable-next-line @typescript-eslint/no-var-requires - const tar = require('tar-fs'); - // eslint-disable-next-line @typescript-eslint/no-var-requires - const bzip = require('unbzip2-stream'); return new Promise((fulfill, reject) => { const tarStream = tar.extract(folderPath); tarStream.on('error', reject); diff --git a/src/node/NodeWebSocketTransport.ts b/src/node/NodeWebSocketTransport.ts index 393763239fe..e32035f65ec 100644 --- a/src/node/NodeWebSocketTransport.ts +++ b/src/node/NodeWebSocketTransport.ts @@ -13,20 +13,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { ConnectionTransport } from '../common/ConnectionTransport.js'; import NodeWebSocket from 'ws'; +import { ConnectionTransport } from '../common/ConnectionTransport.js'; +import { packageVersion } from '../generated/version.js'; export class NodeWebSocketTransport implements ConnectionTransport { static create(url: string): Promise { - // eslint-disable-next-line @typescript-eslint/no-var-requires - const pkg = require('../../../../package.json'); return new Promise((resolve, reject) => { const ws = new NodeWebSocket(url, [], { followRedirects: true, perMessageDeflate: false, maxPayload: 256 * 1024 * 1024, // 256Mb headers: { - 'User-Agent': `Puppeteer ${pkg.version}`, + 'User-Agent': `Puppeteer ${packageVersion}`, }, }); diff --git a/src/templates/version.ts.tmpl b/src/templates/version.ts.tmpl new file mode 100644 index 00000000000..18444ac3b50 --- /dev/null +++ b/src/templates/version.ts.tmpl @@ -0,0 +1 @@ +export const packageVersion = 'PACKAGE_VERSION'; diff --git a/utils/apply_next_version.js b/utils/apply_next_version.js index cd62839d2a6..2882eed45b7 100644 --- a/utils/apply_next_version.js +++ b/utils/apply_next_version.js @@ -12,7 +12,6 @@ const current_sha = execSync(`git rev-parse HEAD`).toString('utf8'); if (upstream_sha.trim() !== current_sha.trim()) { console.log('REFUSING TO PUBLISH: this is not tip-of-tree!'); process.exit(1); - return; } const package = require('../package.json'); diff --git a/utils/generate_version_file.js b/utils/generate_version_file.js new file mode 100644 index 00000000000..a525801ef4a --- /dev/null +++ b/utils/generate_version_file.js @@ -0,0 +1,9 @@ +const { writeFileSync, readFileSync } = require('fs'); +const { join } = require('path'); + +writeFileSync( + join(__dirname, '../src/generated/version.ts'), + readFileSync(join(__dirname, '../src/templates/version.ts.tmpl'), { + encoding: 'utf-8', + }).replace('PACKAGE_VERSION', require('../package.json').version) +); diff --git a/utils/prepare_puppeteer_core.js b/utils/prepare_puppeteer_core.js index e1e9a64f2f4..e476b39fe4a 100755 --- a/utils/prepare_puppeteer_core.js +++ b/utils/prepare_puppeteer_core.js @@ -21,7 +21,11 @@ const path = require('path'); const packagePath = path.join(__dirname, '..', 'package.json'); const json = require(packagePath); -json.name = 'puppeteer-core'; delete json.scripts.install; + +json.name = 'puppeteer-core'; json.main = './cjs-entry-core.js'; +json.exports['.'].import = './lib/esm/puppeteer/node-puppeteer-core.js'; +json.exports['.'].require = './cjs-entry-core.js'; + fs.writeFileSync(packagePath, JSON.stringify(json, null, ' '));