From 88e0997b3997f8067e830907d5be17d84b8d0885 Mon Sep 17 00:00:00 2001 From: jrandolf <101637635+jrandolf@users.noreply.github.com> Date: Tue, 19 Sep 2023 17:13:30 +0200 Subject: [PATCH] chore: implement `@puppeteer/doctest` (#10933) --- .eslintignore | 1 + .github/workflows/changed-packages.yml | 1 + .github/workflows/ci.yml | 20 ++ .gitignore | 1 + .prettierignore | 1 + package-lock.json | 348 +++++++++++++++++++++++- package.json | 11 +- tools/doctest/package.json | 41 +++ tools/doctest/src/doctest.ts | 359 +++++++++++++++++++++++++ tools/doctest/tsconfig.json | 11 + 10 files changed, 785 insertions(+), 9 deletions(-) create mode 100644 tools/doctest/package.json create mode 100644 tools/doctest/src/doctest.ts create mode 100644 tools/doctest/tsconfig.json diff --git a/.eslintignore b/.eslintignore index 0783d2e0..29d33afa 100644 --- a/.eslintignore +++ b/.eslintignore @@ -5,6 +5,7 @@ node_modules # Production build/ lib/ +bin/ # Generated files **/*.tsbuildinfo diff --git a/.github/workflows/changed-packages.yml b/.github/workflows/changed-packages.yml index 51b5ced6..15044bf1 100644 --- a/.github/workflows/changed-packages.yml +++ b/.github/workflows/changed-packages.yml @@ -43,6 +43,7 @@ jobs: - 'test-d/**' - 'tools/mochaRunner/**' - '.mocharc.cjs' + - 'tools/doctest/**' website: - '.github/workflows/ci.yml' - 'docs/**' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 391cb11e..fcaf4d51 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -111,6 +111,26 @@ jobs: curl -H "Content-Type: application/json" -X POST --user "$CRAWLER_USER_ID:$CRAWLER_API_KEY" \ "https://crawler.algolia.com/api/1/crawlers/$CRAWLER_ID/reindex" + doctest: + name: Doctest + runs-on: ubuntu-latest + needs: check-changes + if: ${{ contains(fromJSON(needs.check-changes.outputs.changes), 'puppeteer') }} + steps: + - name: Check out repository + uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + - name: Set up Node.js + uses: actions/setup-node@5e21ff4d9bc1a8cf6de233a3057d20ec6b3fb69d # v3.8.1 + with: + cache: npm + node-version: lts/* + - name: Install dependencies + run: npm ci + env: + PUPPETEER_SKIP_DOWNLOAD: true + - name: Run tests + run: npm run doctest + chrome-tests: name: ${{ matrix.suite }} tests on ${{ matrix.os }} runs-on: ${{ matrix.os }} diff --git a/.gitignore b/.gitignore index 83af90e0..ff9659ba 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ node_modules # Production build/ lib/ +bin/ # Generated files **/*.tsbuildinfo diff --git a/.prettierignore b/.prettierignore index 8196eff6..f09db225 100644 --- a/.prettierignore +++ b/.prettierignore @@ -5,6 +5,7 @@ node_modules # Production build/ lib/ +bin/ # Generated files **/*.tsbuildinfo diff --git a/package-lock.json b/package-lock.json index 319a5b78..805c21bf 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,8 @@ "packages/*", "test", "test/installation", - "tools/eslint" + "tools/eslint", + "tools/doctest" ], "devDependencies": { "@actions/core": "1.10.0", @@ -1849,6 +1850,10 @@ "resolved": "packages/browsers", "link": true }, + "node_modules/@puppeteer/doctest": { + "resolved": "tools/doctest", + "link": true + }, "node_modules/@puppeteer/eslint": { "resolved": "tools/eslint", "link": true @@ -2172,6 +2177,197 @@ "integrity": "sha512-sXXKG+uL9IrKqViTtao2Ws6dy0znu9sOaP1di/jKGW1M6VssO8vlpXCQcpZ+jisQ1tTFAC5Jo/EOzFbggBagFQ==", "dev": true }, + "node_modules/@swc/core": { + "version": "1.3.85", + "resolved": "https://registry.npmjs.org/@swc/core/-/core-1.3.85.tgz", + "integrity": "sha512-qnoxp+2O0GtvRdYnXgR1v8J7iymGGYpx6f6yCK9KxipOZOjrlKILFANYlghQxZyPUfXwK++TFxfSlX4r9wK+kg==", + "hasInstallScript": true, + "dependencies": { + "@swc/types": "^0.1.4" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/swc" + }, + "optionalDependencies": { + "@swc/core-darwin-arm64": "1.3.85", + "@swc/core-darwin-x64": "1.3.85", + "@swc/core-linux-arm-gnueabihf": "1.3.85", + "@swc/core-linux-arm64-gnu": "1.3.85", + "@swc/core-linux-arm64-musl": "1.3.85", + "@swc/core-linux-x64-gnu": "1.3.85", + "@swc/core-linux-x64-musl": "1.3.85", + "@swc/core-win32-arm64-msvc": "1.3.85", + "@swc/core-win32-ia32-msvc": "1.3.85", + "@swc/core-win32-x64-msvc": "1.3.85" + }, + "peerDependencies": { + "@swc/helpers": "^0.5.0" + }, + "peerDependenciesMeta": { + "@swc/helpers": { + "optional": true + } + } + }, + "node_modules/@swc/core-darwin-arm64": { + "version": "1.3.85", + "resolved": "https://registry.npmjs.org/@swc/core-darwin-arm64/-/core-darwin-arm64-1.3.85.tgz", + "integrity": "sha512-jTikp+i4nO4Ofe6qGm4I3sFeebD1OvueBCHITux5tQKD6umN1c2z4CRGv6K49NIz/qEpUcdr6Qny6K+3yibVFQ==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=10" + } + }, + "node_modules/@swc/core-darwin-x64": { + "version": "1.3.85", + "resolved": "https://registry.npmjs.org/@swc/core-darwin-x64/-/core-darwin-x64-1.3.85.tgz", + "integrity": "sha512-3uHYkjVU+2F+YbVYtq5rH0uCJIztFTALaS3mQEfQUZKXZ5/8jD5titTCRqFKtSlQg0CzaFZgsYsuqwYBmgN0mA==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=10" + } + }, + "node_modules/@swc/core-linux-arm-gnueabihf": { + "version": "1.3.85", + "resolved": "https://registry.npmjs.org/@swc/core-linux-arm-gnueabihf/-/core-linux-arm-gnueabihf-1.3.85.tgz", + "integrity": "sha512-ouHzAHsFaEOkRuoTAOI/8n2m8BQAAnb4vr/xbMhhDOmix0lp5eNsW5Iac/EcJ2uG6B3n7P2K8oycj9SWkj+pfw==", + "cpu": [ + "arm" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=10" + } + }, + "node_modules/@swc/core-linux-arm64-gnu": { + "version": "1.3.85", + "resolved": "https://registry.npmjs.org/@swc/core-linux-arm64-gnu/-/core-linux-arm64-gnu-1.3.85.tgz", + "integrity": "sha512-/Z1CZOWiO+NqJEh1J20PIxQFHMH43upQJ1l7FJ5Z7+MyuYF8WkeJ7OSovau729pBR+38vvvccEJrMZIztfv7hQ==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=10" + } + }, + "node_modules/@swc/core-linux-arm64-musl": { + "version": "1.3.85", + "resolved": "https://registry.npmjs.org/@swc/core-linux-arm64-musl/-/core-linux-arm64-musl-1.3.85.tgz", + "integrity": "sha512-gfh7CfKavi076dbMBTzfdawSGcYfZ4+1Q+8aRkSesqepKHcIWIJti8Cf3zB4a6CHNhJe+VN0Gb7DEfumydAm1w==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=10" + } + }, + "node_modules/@swc/core-linux-x64-gnu": { + "version": "1.3.85", + "resolved": "https://registry.npmjs.org/@swc/core-linux-x64-gnu/-/core-linux-x64-gnu-1.3.85.tgz", + "integrity": "sha512-lWVqjHKzofb9q1qrBM4dLqO7CIisp08/xMS5Hz9DWex1gTc5F2b6yJO6Ceqwa256GMweJcdP6A5EvEFQAiZ5dg==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=10" + } + }, + "node_modules/@swc/core-linux-x64-musl": { + "version": "1.3.85", + "resolved": "https://registry.npmjs.org/@swc/core-linux-x64-musl/-/core-linux-x64-musl-1.3.85.tgz", + "integrity": "sha512-EPJmlfqC05TUetnlErxNRyIp7Nc3B2w9abET6oQ/EgldeAeQnZ3M6svMViET/c2QSomgrU3rdP+Qcozkt62/4A==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=10" + } + }, + "node_modules/@swc/core-win32-arm64-msvc": { + "version": "1.3.85", + "resolved": "https://registry.npmjs.org/@swc/core-win32-arm64-msvc/-/core-win32-arm64-msvc-1.3.85.tgz", + "integrity": "sha512-ibckJDZw8kNosciMexwk0z75ZyUhwtiFMV9rSBpup0opa7NNCUCoERCJ1e9LRyMdhsVUoLpZg/KZiHCdTw96hQ==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=10" + } + }, + "node_modules/@swc/core-win32-ia32-msvc": { + "version": "1.3.85", + "resolved": "https://registry.npmjs.org/@swc/core-win32-ia32-msvc/-/core-win32-ia32-msvc-1.3.85.tgz", + "integrity": "sha512-hY4MpHGUVQHL1T2kgRXOigDho4DTIpVPYzJ4uyy8VQRbS7GzN5XtvdGP/fA4zp8+2BQjcig+6J7Y92SY15ouNQ==", + "cpu": [ + "ia32" + ], + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=10" + } + }, + "node_modules/@swc/core-win32-x64-msvc": { + "version": "1.3.85", + "resolved": "https://registry.npmjs.org/@swc/core-win32-x64-msvc/-/core-win32-x64-msvc-1.3.85.tgz", + "integrity": "sha512-ktxWOMFJ0iqKn6WUHtXqi4CS7xkyHmrRtjllGRuGqxmLmDX/HSOfuQ55Tm1KXKk5oHLacJkUbOSF2kBrpZ8dpg==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=10" + } + }, + "node_modules/@swc/types": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/@swc/types/-/types-0.1.4.tgz", + "integrity": "sha512-z/G02d+59gyyUb7KYhKi9jOhicek6QD2oMaotUyG+lUkybpXoV49dY9bj7Ah5Q+y7knK2jU67UTX9FyfGzaxQg==" + }, "node_modules/@tootallnate/once": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz", @@ -2262,6 +2458,12 @@ "integrity": "sha512-amrLbRqTU9bXMCc6uX0sWpxsQzRIo9z6MJPkH1pkez/qOxuqSZVuryJAWoBRq94CeG8JxY+VK4Le9HtjQR5T9A==", "dev": true }, + "node_modules/@types/doctrine": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/@types/doctrine/-/doctrine-0.0.6.tgz", + "integrity": "sha512-KlEqPtaNBHBJ2/fVA4yLdD0Tc8zw34pKU4K5SHBIEwtLJ8xxumIC1xeG+4S+/9qhVj2MqC7O3Ld8WvDG4HqlgA==", + "dev": true + }, "node_modules/@types/eslint": { "version": "8.44.2", "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-8.44.2.tgz", @@ -2406,6 +2608,15 @@ "integrity": "sha512-9GcLXF0/v3t80caGs5p2rRfkB+a8VBGLJZVih6CNFkx8IZ994wiKKLSRs9nuFwk1HevWs/1mnUmkApGrSGsShA==", "dev": true }, + "node_modules/@types/source-map-support": { + "version": "0.5.7", + "resolved": "https://registry.npmjs.org/@types/source-map-support/-/source-map-support-0.5.7.tgz", + "integrity": "sha512-rJqBfLel8jPuL5MwXxMH2Cdb6D80Snu3YJxDE+VJAmtT04l7j3OA7h+FYXlYDys0WeBVH/MPbExj3B8NCaDw9g==", + "dev": true, + "dependencies": { + "source-map": "^0.6.0" + } + }, "node_modules/@types/stack-utils": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-2.0.1.tgz", @@ -2687,7 +2898,6 @@ "version": "8.10.0", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.10.0.tgz", "integrity": "sha512-F0SAmZ8iUtS//m8DmCTA0jlh6TDKkHQyK6xc6V4KDTyZKA9dnvX9/3sRTVQrWm79glUAZbnmmNcdYwUIHWVybw==", - "dev": true, "bin": { "acorn": "bin/acorn" }, @@ -3188,8 +3398,7 @@ "node_modules/buffer-from": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", - "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", - "dev": true + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==" }, "node_modules/builtin-modules": { "version": "3.3.0", @@ -4045,7 +4254,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", "integrity": "sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==", - "dev": true, "dependencies": { "esutils": "^2.0.2" }, @@ -8882,6 +9090,96 @@ "node": ">=12.13.0" } }, + "node_modules/pkg-dir": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-7.0.0.tgz", + "integrity": "sha512-Ie9z/WINcxxLp27BKOCHGde4ITq9UklYKDzVo1nhk5sqGEXU3FpkwP5GM2voTGJkGd9B3Otl+Q4uwSOeSUtOBA==", + "dependencies": { + "find-up": "^6.3.0" + }, + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/pkg-dir/node_modules/find-up": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-6.3.0.tgz", + "integrity": "sha512-v2ZsoEuVHYy8ZIlYqwPe/39Cy+cFDzp4dXPaxNvkEuouymu+2Jbz0PxpKarJHYJTmv2HWT3O382qY8l4jMWthw==", + "dependencies": { + "locate-path": "^7.1.0", + "path-exists": "^5.0.0" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/pkg-dir/node_modules/locate-path": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-7.2.0.tgz", + "integrity": "sha512-gvVijfZvn7R+2qyPX8mAuKcFGDf6Nc61GdvGafQsHL0sBIxfKzA+usWn4GFC/bk+QdwPUD4kWFJLhElipq+0VA==", + "dependencies": { + "p-locate": "^6.0.0" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/pkg-dir/node_modules/p-limit": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-4.0.0.tgz", + "integrity": "sha512-5b0R4txpzjPWVw/cXXUResoD4hb6U/x9BH08L7nw+GN1sezDzPdxeRvpc9c433fZhBan/wusjbCsqwqm4EIBIQ==", + "dependencies": { + "yocto-queue": "^1.0.0" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/pkg-dir/node_modules/p-locate": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-6.0.0.tgz", + "integrity": "sha512-wPrq66Llhl7/4AGC6I+cqxT07LhXvWL08LNXz1fENOw0Ap4sRZZ/gZpTTJ5jpurzzzfS2W/Ge9BY3LgLjCShcw==", + "dependencies": { + "p-limit": "^4.0.0" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/pkg-dir/node_modules/path-exists": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-5.0.0.tgz", + "integrity": "sha512-RjhtfwJOxzcFmNOi6ltcbcu4Iu+FL3zEj83dk4kAS+fVpTxXLO1b38RvJgT/0QwvV/L3aY9TAnyv0EOqW4GoMQ==", + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + } + }, + "node_modules/pkg-dir/node_modules/yocto-queue": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.0.0.tgz", + "integrity": "sha512-9bnSc/HEW2uRy67wc+T8UwauLuPJVn28jb+GtJY16iiKWyvmYJRXVT4UamsAEGQfPohgr2q4Tq0sQbQlxTfi1g==", + "engines": { + "node": ">=12.20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/plur": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/plur/-/plur-4.0.0.tgz", @@ -9982,7 +10280,6 @@ "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", - "devOptional": true, "engines": { "node": ">=0.10.0" } @@ -9991,7 +10288,6 @@ "version": "0.5.21", "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.21.tgz", "integrity": "sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==", - "dev": true, "dependencies": { "buffer-from": "^1.0.0", "source-map": "^0.6.0" @@ -11200,7 +11496,6 @@ "version": "17.7.2", "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", - "dev": true, "dependencies": { "cliui": "^8.0.1", "escalade": "^3.1.1", @@ -11471,6 +11766,43 @@ "mocha": "10.2.0" } }, + "tools/doctest": { + "name": "@puppeteer/doctest", + "version": "0.1.0", + "license": "Apache-2.0", + "dependencies": { + "@swc/core": "1.3.85", + "acorn": "8.10.0", + "doctrine": "3.0.0", + "glob": "10.3.4", + "pkg-dir": "7.0.0", + "source-map": "0.7.4", + "source-map-support": "0.5.21", + "yargs": "17.7.2" + }, + "bin": { + "doctest": "bin/doctest.js" + }, + "devDependencies": { + "@types/doctrine": "0.0.6", + "@types/node": "20.6.2", + "@types/source-map-support": "0.5.7" + } + }, + "tools/doctest/node_modules/@types/node": { + "version": "20.6.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.6.2.tgz", + "integrity": "sha512-Y+/1vGBHV/cYk6OI1Na/LHzwnlNCAfU3ZNGrc1LdRe/LAIbdDPTTv/HU3M7yXN448aTVDq3eKRm2cg7iKLb8gw==", + "dev": true + }, + "tools/doctest/node_modules/source-map": { + "version": "0.7.4", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.4.tgz", + "integrity": "sha512-l3BikUxvPOcn5E74dZiq5BGsTb5yEwhaTSzccU6t4sDOH8NWJCstKO5QT2CvtFoK6F0saL7p9xHAqHOlCPJygA==", + "engines": { + "node": ">= 8" + } + }, "tools/eslint": { "name": "@puppeteer/eslint", "version": "0.1.0", diff --git a/package.json b/package.json index 26f8fb3a..bad01a01 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ "lint": "run-s lint:prettier lint:eslint", "postinstall": "npm run postinstall --workspaces --if-present", "prepare": "npm run prepare --workspaces --if-present", + "doctest": "wireit", "test-install": "npm run test --workspace @puppeteer-test/installation", "test-types": "tsd -t packages/puppeteer", "test:chrome:headful": "wireit", @@ -59,6 +60,13 @@ "./packages/puppeteer-core:build:docs" ] }, + "doctest": { + "command": "npx ./tools/doctest 'packages/puppeteer-core/lib/esm/**/*.js'", + "dependencies": [ + "./packages/puppeteer-core:build", + "./tools/doctest:build" + ] + }, "test:chrome:headful": { "command": "npm test -- --test-suite chrome-headful" }, @@ -184,6 +192,7 @@ "packages/*", "test", "test/installation", - "tools/eslint" + "tools/eslint", + "tools/doctest" ] } diff --git a/tools/doctest/package.json b/tools/doctest/package.json new file mode 100644 index 00000000..d0ec810f --- /dev/null +++ b/tools/doctest/package.json @@ -0,0 +1,41 @@ +{ + "name": "@puppeteer/doctest", + "version": "0.1.0", + "type": "module", + "private": true, + "bin": "./bin/doctest.js", + "description": "Tests JSDoc @example code within a file.", + "license": "Apache-2.0", + "scripts": { + "build": "wireit", + "clean": "../clean.js" + }, + "wireit": { + "build": { + "command": "tsc -b && chmod +x ./bin/doctest.js", + "clean": "if-file-deleted", + "files": [ + "src/**" + ], + "output": [ + "bin/**", + "tsconfig.tsbuildinfo" + ] + } + }, + "dependencies": { + "@swc/core": "1.3.85", + "acorn": "8.10.0", + "doctrine": "3.0.0", + "glob": "10.3.4", + "pkg-dir": "7.0.0", + "source-map": "0.7.4", + "source-map-support": "0.5.21", + "yargs": "17.7.2" + }, + "devDependencies": { + "@types/doctrine": "0.0.6", + "@types/node": "20.6.2", + "@types/source-map-support": "0.5.7" + } +} diff --git a/tools/doctest/src/doctest.ts b/tools/doctest/src/doctest.ts new file mode 100644 index 00000000..e78cbf2f --- /dev/null +++ b/tools/doctest/src/doctest.ts @@ -0,0 +1,359 @@ +#! /usr/bin/env -S node --test-reporter spec + +/** + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * `@puppeteer/doctest` tests `@example` code within a JavaScript file. + * + * There are a few reasonable assumptions for this tool to work: + * + * 1. Examples are written in block comments, not line comments. + * 2. Examples do not use packages that are not available to the file it exists + * in. (Note the package will always be available). + * 3. Examples are strictly written between code fences (\`\`\`) on separate + * lines. For example, \`\`\`console.log(1)\`\`\` is not allowed. + * 4. Code is written using ES modules. + * + * By default, code blocks are interpreted as JavaScript. Use \`\`\`ts to change + * the language. In general, the format is "\`\`\`[language] [ignore] [fail]". + * + * If there are several code blocks within an example, they are concatenated. + */ +import 'source-map-support/register.js'; + +import assert from 'node:assert'; +import {createHash} from 'node:crypto'; +import {mkdtemp, readFile, rm, writeFile} from 'node:fs/promises'; +import {basename, dirname, join, relative, resolve} from 'node:path'; +import {test} from 'node:test'; +import {pathToFileURL} from 'node:url'; + +import {transform, type Output} from '@swc/core'; +import {parse as parseJs} from 'acorn'; +import {parse, type Tag} from 'doctrine'; +import {Glob} from 'glob'; +import {packageDirectory} from 'pkg-dir'; +import { + SourceMapConsumer, + SourceMapGenerator, + type RawSourceMap, +} from 'source-map'; +import yargs from 'yargs'; +import {hideBin} from 'yargs/helpers'; + +// This is 1-indexed. +interface Position { + line: number; + column: number; +} + +interface Comment { + file: string; + text: string; + position: Position; +} + +interface ExtractedSourceLocation { + // File path to the original source code. + origin: string; + // Mappings from the extracted code to the original code. + positions: Array<{ + // The 1-indexed line number for the extracted code. + extracted: number; + // The position in the original code. + original: Position; + }>; +} + +interface ExampleCode extends ExtractedSourceLocation { + language: Language; + code: string; + fail: boolean; +} + +const enum Language { + JavaScript, + TypeScript, +} + +const CODE_FENCE = '```'; +const BLOCK_COMMENT_START = ' * '; + +const {files = []} = await yargs(hideBin(process.argv)) + .scriptName('@puppeteer/doctest') + .command('* ', `JSDoc @example code tester.`) + .positional('files', { + describe: 'Files to test', + type: 'string', + }) + .array('files') + .version(false) + .help() + .parse(); + +for await (const file of new Glob(files, {})) { + void test(file, async context => { + const testDirectory = await createTestDirectory(file); + context.after(async () => { + if (!process.env['KEEP_TESTS']) { + await rm(testDirectory, {force: true, recursive: true}); + } + }); + const tests = []; + for (const example of await extractJSDocComments(file).then( + extractExampleCode + )) { + tests.push( + context.test( + `${file}:${example.positions[0]!.original.line}:${ + example.positions[0]!.original.column + }`, + async () => { + await run(testDirectory, example); + } + ) + ); + } + await Promise.all(tests); + }); +} + +async function createTestDirectory(file: string) { + const dir = await packageDirectory({cwd: dirname(file)}); + if (!dir) { + throw new Error(`Could not find package root for ${file}.`); + } + + return await mkdtemp(join(dir, 'doctest-')); +} + +async function run(tempdir: string, example: Readonly) { + const path = getTestPath(tempdir, example.code); + await compile(example.language, example.code, path, example); + try { + await import(pathToFileURL(path).toString()); + if (example.fail) { + throw new Error(`Expected failure.`); + } + } catch (error) { + if (!example.fail) { + throw error; + } + } +} + +function getTestPath(dir: string, code: string) { + return join( + dir, + `doctest-${createHash('md5').update(code).digest('hex')}.js` + ); +} + +async function compile( + language: Language, + sourceCode: string, + filePath: string, + location: ExtractedSourceLocation +) { + const output = await compileCode(language, sourceCode); + const map = await getExtractSourceMap(output.map, filePath, location); + await writeFile(filePath, inlineSourceMap(output.code, map)); +} + +function inlineSourceMap(code: string, sourceMap: RawSourceMap) { + return `${code}\n//# sourceMappingURL=data:application/json;base64,${Buffer.from( + JSON.stringify(sourceMap) + ).toString('base64')}`; +} + +async function getExtractSourceMap( + map: string, + generatedFile: string, + location: ExtractedSourceLocation +) { + const sourceMap = JSON.parse(map) as RawSourceMap; + sourceMap.file = basename(generatedFile); + sourceMap.sourceRoot = ''; + sourceMap.sources = [ + relative(dirname(generatedFile), resolve(location.origin)), + ]; + const consumer = await new SourceMapConsumer(sourceMap); + const generator = new SourceMapGenerator({ + file: consumer.file, + sourceRoot: consumer.sourceRoot, + }); + // We want descending order of the `generated` property. + const positions = [...location.positions].reverse(); + consumer.eachMapping(mapping => { + // Note `mapping.originalLine` is the line number with respect to the + // extracted, raw code. + const {extracted, original} = positions.find(({extracted}) => { + return mapping.originalLine >= extracted; + })!; + + // `original.line` will account for `extracted`, so we need to subtract + // `extracted` to avoid duplicity. We also subtract 1 because `extracted` is + // 1-indexed. + mapping.originalLine -= extracted - 1; + + generator.addMapping({ + ...mapping, + original: { + line: mapping.originalLine + original.line - 1, + column: mapping.originalColumn + original.column - 1, + }, + generated: { + line: mapping.generatedLine, + column: mapping.generatedColumn, + }, + }); + }); + return generator.toJSON(); +} + +const LANGUAGE_TO_SYNTAX = { + [Language.TypeScript]: 'typescript', + [Language.JavaScript]: 'ecmascript', +} as const; + +async function compileCode(language: Language, code: string) { + return (await transform(code, { + sourceMaps: true, + inlineSourcesContent: false, + jsc: { + parser: { + syntax: LANGUAGE_TO_SYNTAX[language], + }, + target: 'es2022', + }, + })) as Required; +} + +const enum Option { + Ignore = 'ignore', + Fail = 'fail', +} + +function* extractExampleCode( + comments: Iterable> +): Iterable> { + interface Context { + language: Language; + fail: boolean; + start: number; + } + for (const {file, text, position: loc} of comments) { + const {tags} = parse(text, { + unwrap: true, + tags: ['example'], + lineNumbers: true, + preserveWhitespace: true, + }); + for (const {description, lineNumber} of tags as Array< + Tag & {lineNumber: number} + >) { + if (!description) { + continue; + } + const lines = description.split('\n'); + const blocks: ExampleCode[] = []; + let context: Context | undefined; + for (let i = 0; i < lines.length; i++) { + const line = lines[i]!; + const borderIndex = line.indexOf(CODE_FENCE); + if (borderIndex === -1) { + continue; + } + if (context) { + blocks.push({ + language: context.language, + code: lines.slice(context.start, i).join('\n'), + origin: file, + positions: [ + { + extracted: 1, + original: { + line: loc.line + lineNumber + context.start, + column: + loc.column + borderIndex + BLOCK_COMMENT_START.length + 1, + }, + }, + ], + fail: context.fail, + }); + context = undefined; + continue; + } + const [tag, ...options] = line + .slice(borderIndex + CODE_FENCE.length) + .split(' '); + if (options.includes(Option.Ignore)) { + // Ignore the code sample. + continue; + } + const fail = options.includes(Option.Fail); + // Code starts on the next line. + const start = i + 1; + if (!tag || tag.match(/js|javascript/)) { + context = {language: Language.JavaScript, fail, start}; + } else if (tag.match(/ts|typescript/)) { + context = {language: Language.TypeScript, fail, start}; + } + } + // Merging the blocks into a single block. + yield blocks.reduce( + (context, {language, code, positions: [position], fail}, index) => { + assert(position); + return { + origin: file, + language: language || context.language, + code: `${context.code}\n${code}`, + positions: [ + ...context.positions, + { + ...position, + extracted: + context.code.split('\n').length + + context.positions.at(-1)!.extracted - + // We subtract this because of the accumulated '\n'. + (index - 1), + }, + ], + fail: fail || context.fail, + }; + } + ); + } + } +} + +async function extractJSDocComments(file: string) { + const contents = await readFile(file, 'utf8'); + const comments: Comment[] = []; + parseJs(contents, { + ecmaVersion: 'latest', + sourceType: 'module', + locations: true, + sourceFile: file, + onComment(isBlock, text, _, __, loc) { + if (isBlock) { + comments.push({file, text, position: loc!}); + } + }, + }); + return comments; +} diff --git a/tools/doctest/tsconfig.json b/tools/doctest/tsconfig.json new file mode 100644 index 00000000..6b822157 --- /dev/null +++ b/tools/doctest/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "./src", + "outDir": "./bin", + "sourceMap": true, + "declaration": false, + "declarationMap": false, + "composite": false + } +}