thunderstrike/puppeteer
thunderstrike/puppeteer
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor: use @puppeteer/browsers for install (#9898)

Browse Source 
Co-authored-by: Nikolay Vitkov <34244704+Lightning00Blade@users.noreply.github.com>
This commit is contained in:
Alex Rudenko 2023-03-28 11:27:50 +02:00 committed by GitHub
parent a88322e947
commit 3936600ba9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 115 additions and 178 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.github/workflows/ci.yml vendored
View File

@ -143,7 +143,7 @@ jobs:
uses: actions/cache@v3
with:
path: ~/.cache/puppeteer/chrome
key: ${{ runner.os }}-chromium-${{ hashFiles('packages/puppeteer-core/src/revisions.ts') }}
key: ${{ runner.os }}-chromium-${{ hashFiles('packages/puppeteer-core/src/revisions.ts') }}-${{ hashFiles('packages/puppeteer/src/node/install.ts') }}
- name: Install Chromium
run: npm run postinstall
- name: Tests types
@ -211,7 +211,7 @@ jobs:
uses: actions/cache@v3
with:
path: ~/.cache/puppeteer/firefox
key: ${{ runner.os }}-firefox-${{ hashFiles('packages/puppeteer-core/src/revisions.ts') }}
key: ${{ runner.os }}-firefox-${{ hashFiles('packages/puppeteer-core/src/revisions.ts') }}-${{ hashFiles('packages/puppeteer/src/node/install.ts') }}
- name: Install Firefox
env:
PUPPETEER_PRODUCT: firefox

8
docker/Dockerfile
View File

@ -17,11 +17,11 @@ USER pptruser
WORKDIR /home/pptruser
COPY puppeteer-latest.tgz puppeteer-core-latest.tgz ./
COPY puppeteer-browsers-latest.tgz puppeteer-latest.tgz puppeteer-core-latest.tgz ./
# Install puppeteer and puppeteer-core into /home/pptruser/node_modules.
RUN npm i ./puppeteer-core-latest.tgz ./puppeteer-latest.tgz \
&& rm ./puppeteer-core-latest.tgz ./puppeteer-latest.tgz \
# Install @puppeteer/browsers, puppeteer and puppeteer-core into /home/pptruser/node_modules.
RUN npm i ./puppeteer-browsers-latest.tgz ./puppeteer-core-latest.tgz ./puppeteer-latest.tgz \
&& rm ./puppeteer-browsers-latest.tgz ./puppeteer-core-latest.tgz ./puppeteer-latest.tgz \
&& (node -e "require('child_process').execSync(require('puppeteer').executablePath() + ' --credits', {stdio: 'inherit'})" > THIRD_PARTY_NOTICES)
CMD ["google-chrome-stable"]

4
docker/pack.sh
View File

@ -7,10 +7,12 @@ set -e
cd docker
npm pack --workspace puppeteer --workspace puppeteer-core --pack-destination .
npm pack --workspace puppeteer --workspace puppeteer-core --workspace @puppeteer/browsers --pack-destination .
rm -f puppeteer-core-latest.tgz
rm -f puppeteer-latest.tgz
rm -f puppeteer-browsers-latest.tgz
mv puppeteer-core-*.tgz puppeteer-core-latest.tgz
mv puppeteer-browsers-*.tgz puppeteer-browsers-latest.tgz
mv puppeteer-[0-9]*.tgz puppeteer-latest.tgz

4
package-lock.json generated
View File

@ -9317,7 +9317,7 @@
},
"packages/browsers": {
"name": "@puppeteer/browsers",
"version": "0.2.0",
"version": "0.3.0",
"license": "Apache-2.0",
"dependencies": {
"debug": "4.3.4",
@ -9470,6 +9470,7 @@
"hasInstallScript": true,
"license": "Apache-2.0",
"dependencies": {
"@puppeteer/browsers": "0.3.0",
"cosmiconfig": "8.1.3",
"https-proxy-agent": "5.0.1",
"progress": "2.0.3",
@ -14452,6 +14453,7 @@
"puppeteer": {
"version": "file:packages/puppeteer",
"requires": {
"@puppeteer/browsers": "0.3.0",
"cosmiconfig": "8.1.3",
"https-proxy-agent": "5.0.1",
"progress": "2.0.3",

4
packages/puppeteer/package.json
View File

@ -81,6 +81,7 @@
"clean": "if-file-deleted",
"dependencies": [
"../puppeteer-core:build",
"../browsers:build",
"generate:package-json"
],
"files": [
@ -119,6 +120,7 @@
"https-proxy-agent": "5.0.1",
"progress": "2.0.3",
"proxy-from-env": "1.1.0",
"puppeteer-core": "19.8.0"
"puppeteer-core": "19.8.0",
"@puppeteer/browsers": "0.3.0"
}
}

236
packages/puppeteer/src/node/install.ts
View File

@ -14,13 +14,15 @@
* limitations under the License.
*/
import https, {RequestOptions} from 'https';
import URL from 'url';
import createHttpsProxyAgent, {HttpsProxyAgentOptions} from 'https-proxy-agent';
import ProgressBar from 'progress';
import {getProxyForUrl} from 'proxy-from-env';
import {PuppeteerNode} from 'puppeteer-core/internal/node/PuppeteerNode.js';
import {
fetch,
Browser,
resolveBuildId,
makeProgressCallback,
detectBrowserPlatform,
BrowserPlatform,
} from '@puppeteer/browsers';
import {Product} from 'puppeteer-core';
import {PUPPETEER_REVISIONS} from 'puppeteer-core/internal/revisions.js';
import {getConfiguration} from '../getConfiguration.js';
@ -29,6 +31,7 @@ import {getConfiguration} from '../getConfiguration.js';
* @internal
*/
const supportedProducts = {
chromium: 'Chromium',
chrome: 'Chromium',
firefox: 'Firefox Nightly',
} as const;
@ -37,170 +40,69 @@ const supportedProducts = {
* @internal
*/
export async function downloadBrowser(): Promise<void> {
overrideProxy();
const configuration = getConfiguration();
if (configuration.skipDownload) {
logPolitely('**INFO** Skipping browser download as instructed.');
return;
}
const puppeteer = new PuppeteerNode({configuration, isPuppeteerCore: false});
let platform = detectBrowserPlatform();
if (!platform) {
throw new Error('The current platform is not supported.');
}
// TODO: remove once Mac ARM is enabled by default for Puppeteer https://github.com/puppeteer/puppeteer/issues/9630.
if (
platform === BrowserPlatform.MAC_ARM &&
!configuration.experiments?.macArmChromiumEnabled
) {
platform = BrowserPlatform.MAC;
}
const product = configuration.defaultProduct!;
const browserFetcher = puppeteer.createBrowserFetcher();
const browser = productToBrowser(product);
let revision = configuration.browserRevision;
// TODO: PUPPETEER_REVISIONS should use Chrome and not Chromium.
const unresolvedBuildId =
configuration.browserRevision ||
PUPPETEER_REVISIONS[product === 'chrome' ? 'chromium' : 'firefox'] ||
'latest';
if (!revision) {
switch (product) {
case 'chrome':
revision = PUPPETEER_REVISIONS.chromium;
break;
case 'firefox':
revision = PUPPETEER_REVISIONS.firefox;
revision = await getFirefoxNightlyVersion();
break;
}
}
const buildId = await resolveBuildId(browser, platform, unresolvedBuildId);
await fetchBinary(revision);
function fetchBinary(revision: string) {
const revisionInfo = browserFetcher.revisionInfo(revision);
// Do nothing if the revision is already downloaded.
if (revisionInfo.local) {
logPolitely(
`${supportedProducts[product]} is already in ${revisionInfo.folderPath}; skipping download.`
);
return;
}
// Override current environment proxy settings with npm configuration, if any.
const NPM_HTTPS_PROXY =
process.env['npm_config_https_proxy'] || process.env['npm_config_proxy'];
const NPM_HTTP_PROXY =
process.env['npm_config_http_proxy'] || process.env['npm_config_proxy'];
const NPM_NO_PROXY = process.env['npm_config_no_proxy'];
if (NPM_HTTPS_PROXY) {
process.env['HTTPS_PROXY'] = NPM_HTTPS_PROXY;
}
if (NPM_HTTP_PROXY) {
process.env['HTTP_PROXY'] = NPM_HTTP_PROXY;
}
if (NPM_NO_PROXY) {
process.env['NO_PROXY'] = NPM_NO_PROXY;
}
function onSuccess(localRevisions: string[]): void {
logPolitely(
`${supportedProducts[product]} (${revisionInfo.revision}) downloaded to ${revisionInfo.folderPath}`
);
const otherRevisions = localRevisions.filter(revision => {
return revision !== revisionInfo.revision;
});
if (otherRevisions.length) {
logPolitely(
`Other installed ${
supportedProducts[product]
} browsers in ${browserFetcher.getDownloadPath()} include: ${otherRevisions.join(
', '
)}. Remove old revisions from ${browserFetcher.getDownloadPath()} if you don't need them.`
);
}
}
function onError(error: Error) {
console.error(
`ERROR: Failed to set up ${supportedProducts[product]} r${revision}! Set "PUPPETEER_SKIP_DOWNLOAD" env variable to skip download.`
);
console.error(error);
process.exit(1);
}
let progressBar: ProgressBar | null = null;
let lastDownloadedBytes = 0;
function onProgress(downloadedBytes: number, totalBytes: number) {
if (!progressBar) {
progressBar = new ProgressBar(
`Downloading ${
supportedProducts[product]
} r${revision} - ${toMegabytes(totalBytes)} [:bar] :percent :etas `,
{
complete: '=',
incomplete: ' ',
width: 20,
total: totalBytes,
}
);
}
const delta = downloadedBytes - lastDownloadedBytes;
lastDownloadedBytes = downloadedBytes;
progressBar.tick(delta);
}
return browserFetcher
.download(revisionInfo.revision, onProgress)
.then(() => {
return browserFetcher.localRevisions();
})
.then(onSuccess)
.catch(onError);
}
function toMegabytes(bytes: number) {
const mb = bytes / 1024 / 1024;
return `${Math.round(mb * 10) / 10} Mb`;
}
async function getFirefoxNightlyVersion(): Promise<string> {
const firefoxVersionsUrl =
'https://product-details.mozilla.org/1.0/firefox_versions.json';
const proxyURL = getProxyForUrl(firefoxVersionsUrl);
const requestOptions: RequestOptions = {};
if (proxyURL) {
const parsedProxyURL = URL.parse(proxyURL);
const proxyOptions = {
...parsedProxyURL,
secureProxy: parsedProxyURL.protocol === 'https:',
} as HttpsProxyAgentOptions;
requestOptions.agent = createHttpsProxyAgent(proxyOptions);
requestOptions.rejectUnauthorized = false;
}
const promise = new Promise<string>((resolve, reject) => {
let data = '';
logPolitely(
`Requesting latest Firefox Nightly version from ${firefoxVersionsUrl}`
);
https
.get(firefoxVersionsUrl, requestOptions, r => {
if (r.statusCode && r.statusCode >= 400) {
return reject(new Error(`Got status code ${r.statusCode}`));
}
r.on('data', chunk => {
data += chunk;
});
r.on('end', () => {
try {
const versions = JSON.parse(data);
return resolve(versions.FIREFOX_NIGHTLY);
} catch {
return reject(new Error('Firefox version not found'));
}
});
})
.on('error', reject);
try {
const result = await fetch({
browser,
cacheDir: configuration.cacheDirectory!,
platform,
buildId,
downloadProgressCallback: makeProgressCallback(browser, buildId),
});
return promise;
logPolitely(
`${supportedProducts[product]} (${result.buildId}) downloaded to ${result.path}`
);
} catch (error) {
console.error(
`ERROR: Failed to set up ${supportedProducts[product]} r${buildId}! Set "PUPPETEER_SKIP_DOWNLOAD" env variable to skip download.`
);
console.error(error);
process.exit(1);
}
}
function productToBrowser(product?: Product) {
switch (product) {
case 'chrome':
return Browser.CHROMIUM;
case 'firefox':
return Browser.FIREFOX;
}
return Browser.CHROMIUM;
}
/**
* @internal
*/
@ -213,3 +115,25 @@ function logPolitely(toBeLogged: unknown): void {
console.log(toBeLogged);
}
}
/**
* @internal
*/
function overrideProxy() {
// Override current environment proxy settings with npm configuration, if any.
const NPM_HTTPS_PROXY =
process.env['npm_config_https_proxy'] || process.env['npm_config_proxy'];
const NPM_HTTP_PROXY =
process.env['npm_config_http_proxy'] || process.env['npm_config_proxy'];
const NPM_NO_PROXY = process.env['npm_config_no_proxy'];
if (NPM_HTTPS_PROXY) {
process.env['HTTPS_PROXY'] = NPM_HTTPS_PROXY;
}
if (NPM_HTTP_PROXY) {
process.env['HTTP_PROXY'] = NPM_HTTP_PROXY;
}
if (NPM_NO_PROXY) {
process.env['NO_PROXY'] = NPM_NO_PROXY;
}
}

12
test/TestExpectations.json
View File

@ -305,12 +305,6 @@
"parameters": ["webDriverBiDi"],
"expectations": ["FAIL", "TIMEOUT"]
},
{
"testIdPattern": "[navigation.spec] navigation Page.goto should not leak listeners during navigation of 11 pages",
"platforms": ["darwin"],
"parameters": ["chrome", "webDriverBiDi"],
"expectations": ["PASS", "TIMEOUT"]
},
{
"testIdPattern": "[navigation.spec] navigation Page.goto should not throw an error for a 404 response with an empty body",
"platforms": ["darwin", "linux", "win32"],
@ -1175,6 +1169,12 @@
"parameters": ["chrome", "webDriverBiDi"],
"expectations": ["FAIL", "PASS", "TIMEOUT"]
},
{
"testIdPattern": "[navigation.spec] navigation Page.goto should not leak listeners during navigation of 11 pages",
"platforms": ["darwin"],
"parameters": ["chrome", "webDriverBiDi"],
"expectations": ["PASS", "TIMEOUT"]
},
{
"testIdPattern": "[navigation.spec] navigation Page.goto should send referer",
"platforms": ["darwin", "linux", "win32"],

5
test/installation/package.json
View File

@ -26,10 +26,11 @@
]
},
"build:packages": {
"command": "npm pack --workspace puppeteer --workspace puppeteer-core",
"command": "npm pack --workspace puppeteer --workspace puppeteer-core --workspace @puppeteer/browsers",
"dependencies": [
"../../packages/puppeteer:build",
"../../packages/puppeteer-core:build"
"../../packages/puppeteer-core:build",
"../../packages/browsers:build"
],
"files": [],
"output": [

3
test/installation/src/constants.ts
View File

@ -22,6 +22,9 @@ import glob from 'glob';
export const PUPPETEER_CORE_PACKAGE_PATH = resolve(
glob.sync('puppeteer-core-*.tgz')[0]!
);
export const PUPPETEER_BROWSERS_PACKAGE_PATH = resolve(
glob.sync('puppeteer-browsers-[0-9]*.tgz')[0]!
);
export const PUPPETEER_PACKAGE_PATH = resolve(
glob.sync('puppeteer-[0-9]*.tgz')[0]!
);

5
test/installation/src/describeInstallation.ts
View File

@ -22,6 +22,7 @@ import {join} from 'path';
import {
PUPPETEER_CORE_PACKAGE_PATH,
PUPPETEER_PACKAGE_PATH,
PUPPETEER_BROWSERS_PACKAGE_PATH,
} from './constants.js';
import {execFile} from './util.js';
@ -78,6 +79,8 @@ export const describeInstallation = (
return PUPPETEER_PACKAGE_PATH;
case 'puppeteer-core':
return PUPPETEER_CORE_PACKAGE_PATH;
case '@puppeteer/browsers':
return PUPPETEER_BROWSERS_PACKAGE_PATH;
default:
return module;
}
@ -102,7 +105,7 @@ export const describeInstallation = (
after(async () => {
if (process.env['KEEP_SANDBOX']) {
await rm(sandbox, {recursive: true, force: true});
await rm(sandbox, {recursive: true, force: true, maxRetries: 5});
}
});

2
test/installation/src/puppeteer-configuration.spec.ts
View File

@ -24,7 +24,7 @@ import {readAsset} from './util.js';
describeInstallation(
'`puppeteer` with configuration',
{
dependencies: ['puppeteer-core', 'puppeteer'],
dependencies: ['puppeteer-core', '@puppeteer/browsers', 'puppeteer'],
before: async cwd => {
await writeFile(
join(cwd, '.puppeteerrc.cjs'),

2
test/installation/src/puppeteer-firefox.spec.ts
View File

@ -24,7 +24,7 @@ import {readAsset} from './util.js';
describeInstallation(
'`puppeteer` with Firefox',
{
dependencies: ['puppeteer-core', 'puppeteer'],
dependencies: ['@puppeteer/browsers', 'puppeteer-core', 'puppeteer'],
env: cwd => {
return {
PUPPETEER_CACHE_DIR: join(cwd, '.cache', 'puppeteer'),

2
test/installation/src/puppeteer-webpack.spec.ts
View File

@ -23,7 +23,7 @@ import {execFile, readAsset} from './util.js';
describeInstallation(
'`puppeteer` with Webpack',
{
dependencies: ['puppeteer-core', 'puppeteer'],
dependencies: ['@puppeteer/browsers', 'puppeteer-core', 'puppeteer'],
devDependencies: ['webpack', 'webpack-cli'],
env: cwd => {
return {

2
test/installation/src/puppeteer.spec.ts
View File

@ -24,7 +24,7 @@ import {readAsset} from './util.js';
describeInstallation(
'`puppeteer`',
{
dependencies: ['puppeteer-core', 'puppeteer'],
dependencies: ['@puppeteer/browsers', 'puppeteer-core', 'puppeteer'],
env: cwd => {
return {
PUPPETEER_CACHE_DIR: join(cwd, '.cache', 'puppeteer'),