github/puppeteer
0
0
Fork 0
mirror of https://github.com/puppeteer/puppeteer synced 2024-06-14 14:02:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: unify working with the cache folder structure (#9681)

Browse Source
This commit is contained in:
Alex Rudenko 2023-02-15 20:41:22 +01:00 committed by GitHub
parent cfb60d01a8
commit 0b27cbe86d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 168 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
packages/browsers/src/CLI.ts
View File

@ -1,9 +1,24 @@
/**
* Copyright 2023 Google Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import yargs from 'yargs';
import ProgressBar from 'progress';
import {hideBin} from 'yargs/helpers';
import {Browser, BrowserPlatform} from './browsers/types.js';
import {fetch} from './fetch.js';
import path from 'path';
type Arguments = {
browser: {
@ -44,11 +59,8 @@ export class CLI {
browser: args.browser.name,
revision: args.browser.revision,
platform: args.platform,
outputDir: path.join(
args.path ?? this.#cachePath,
args.browser.name
),
progressCallback: this.#makeProgressBar(
cacheDir: args.path ?? this.#cachePath,
downloadProgressCallback: this.#makeProgressCallback(
args.browser.name,
args.browser.revision
),
@ -82,8 +94,8 @@ export class CLI {
return `${Math.round(mb * 10) / 10} Mb`;
}
#makeProgressBar(browser: Browser, revision: string) {
let progressBar: ProgressBar | null = null;
#makeProgressCallback(browser: Browser, revision: string) {
let progressBar: ProgressBar;
let lastDownloadedBytes = 0;
return (downloadedBytes: number, totalBytes: number) => {
if (!progressBar) {

52
packages/browsers/src/CacheStructure.ts Normal file
View File

@ -0,0 +1,52 @@
/**
* Copyright 2023 Google Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {Browser, BrowserPlatform} from './browsers/types.js';
import path from 'path';
/**
* The cache used by Puppeteer relies on the following structure:
*
* - rootDir
* -- <browser1> | browserRoot(browser1)
* ---- <platform>-<revision> | installationDir()
* ------ the browser-platform-revision
* ------ specific structure.
* -- <browser2> | browserRoot(browser2)
* ---- <platform>-<revision> | installationDir()
* ------ the browser-platform-revision
* ------ specific structure.
* @internal
*/
export class CacheStructure {
#rootDir: string;
constructor(rootDir: string) {
this.#rootDir = rootDir;
}
browserRoot(browser: Browser): string {
return path.join(this.#rootDir, browser);
}
installationDir(
browser: Browser,
platform: BrowserPlatform,
revision: string
): string {
return path.join(this.browserRoot(browser), `${platform}-${revision}`);
}
}

16
packages/browsers/src/browsers.ts
View File

@ -1,5 +1,21 @@
#!/usr/bin/env node
/**
* Copyright 2023 Google Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {CLI} from './CLI.js';
new CLI().run(process.argv);

4
packages/browsers/src/browsers/browsers.ts
View File

@ -24,8 +24,8 @@ export const downloadUrls = {
};
export const executablePathByBrowser = {
[Browser.CHROME]: chrome.executablePath,
[Browser.FIREFOX]: firefox.executablePath,
[Browser.CHROME]: chrome.relativeExecutablePath,
[Browser.FIREFOX]: firefox.relativeExecutablePath,
};
export {Browser, BrowserPlatform};

18
packages/browsers/src/browsers/chrome.ts
View File

@ -57,26 +57,18 @@ export function resolveDownloadUrl(
)}.zip`;
}
export function executablePath(
export function relativeExecutablePath(
platform: BrowserPlatform,
revision: string,
basePath = ''
_revision: string
): string {
const browserPath = path.join(basePath, `${platform}-${revision}`);
switch (platform) {
case BrowserPlatform.MAC:
case BrowserPlatform.MAC_ARM:
return path.join(
browserPath,
'Chromium.app',
'Contents',
'MacOS',
'Chromium'
);
return path.join('Chromium.app', 'Contents', 'MacOS', 'Chromium');
case BrowserPlatform.LINUX:
return path.join(browserPath, 'chrome');
return 'chrome';
case BrowserPlatform.WIN32:
case BrowserPlatform.WIN64:
return path.join(browserPath, 'chrome.exe');
return 'chrome.exe';
}
}

18
packages/browsers/src/browsers/firefox.ts
View File

@ -38,26 +38,18 @@ export function resolveDownloadUrl(
return `${baseUrl}/${archive(platform, revision)}`;
}
export function executablePath(
export function relativeExecutablePath(
platform: BrowserPlatform,
revision: string,
basePath = ''
_revision: string
): string {
const browserPath = path.join(basePath, `${platform}-${revision}`);
switch (platform) {
case BrowserPlatform.MAC_ARM:
case BrowserPlatform.MAC:
return path.join(
browserPath,
'Firefox Nightly.app',
'Contents',
'MacOS',
'firefox'
);
return path.join('Firefox Nightly.app', 'Contents', 'MacOS', 'firefox');
case BrowserPlatform.LINUX:
return path.join(browserPath, 'firefox', 'firefox');
return path.join('firefox', 'firefox');
case BrowserPlatform.WIN32:
case BrowserPlatform.WIN64:
return path.join(browserPath, 'firefox', 'firefox.exe');
return path.join('firefox', 'firefox.exe');
}
}

2
packages/browsers/src/detectPlatform.ts
View File

@ -17,7 +17,7 @@
import os from 'os';
import {BrowserPlatform} from './browsers/browsers.js';
export function detectPlatform(): BrowserPlatform | undefined {
export function detectBrowserPlatform(): BrowserPlatform | undefined {
const platform = os.platform();
switch (platform) {
case 'darwin':

33
packages/browsers/src/fetch.ts
View File

@ -24,7 +24,8 @@ import {Browser, BrowserPlatform, downloadUrls} from './browsers/browsers.js';
import {downloadFile, headHttpRequest} from './httpUtil.js';
import assert from 'assert';
import {unpackArchive} from './fileUtil.js';
import {detectPlatform} from './detectPlatform.js';
import {detectBrowserPlatform} from './detectPlatform.js';
import {CacheStructure} from './CacheStructure.js';
const debugFetch = debug('puppeteer:browsers:fetcher');
@ -35,7 +36,7 @@ export interface Options {
/**
* Determines the path to download browsers to.
*/
outputDir: string;
cacheDir: string;
/**
* Determines which platform the browser will be suited for.
*
@ -54,7 +55,10 @@ export interface Options {
/**
* Provides information about the progress of the download.
*/
progressCallback?: (downloadedBytes: number, totalBytes: number) => void;
downloadProgressCallback?: (
downloadedBytes: number,
totalBytes: number
) => void;
}
export type InstalledBrowser = {
@ -65,7 +69,7 @@ export type InstalledBrowser = {
};
export async function fetch(options: Options): Promise<InstalledBrowser> {
options.platform ??= detectPlatform();
options.platform ??= detectBrowserPlatform();
if (!options.platform) {
throw new Error(
`Cannot download a binary for the provided platform: ${os.platform()} (${os.arch()})`
@ -78,10 +82,16 @@ export async function fetch(options: Options): Promise<InstalledBrowser> {
);
const fileName = url.toString().split('/').pop();
assert(fileName, `A malformed download URL was found: ${url}.`);
const archivePath = path.join(options.outputDir, fileName);
const outputPath = path.resolve(
options.outputDir,
`${options.platform}-${options.revision}`
const structure = new CacheStructure(options.cacheDir);
const browserRoot = structure.browserRoot(options.browser);
const archivePath = path.join(browserRoot, fileName);
if (!existsSync(browserRoot)) {
await mkdir(browserRoot, {recursive: true});
}
const outputPath = structure.installationDir(
options.browser,
options.platform,
options.revision
);
if (existsSync(outputPath)) {
return {
@ -91,12 +101,9 @@ export async function fetch(options: Options): Promise<InstalledBrowser> {
revision: options.revision,
};
}
if (!existsSync(options.outputDir)) {
await mkdir(options.outputDir, {recursive: true});
}
try {
debugFetch(`Downloading binary from ${url}`);
await downloadFile(url, archivePath, options.progressCallback);
await downloadFile(url, archivePath, options.downloadProgressCallback);
debugFetch(`Installing ${archivePath} to ${outputPath}`);
await unpackArchive(archivePath, outputPath);
} finally {
@ -113,7 +120,7 @@ export async function fetch(options: Options): Promise<InstalledBrowser> {
}
export async function canFetch(options: Options): Promise<boolean> {
options.platform ??= detectPlatform();
options.platform ??= detectBrowserPlatform();
if (!options.platform) {
throw new Error(
`Cannot download a binary for the provided platform: ${os.platform()} (${os.arch()})`

18
packages/browsers/src/launcher.ts
View File

@ -19,8 +19,10 @@ import {
BrowserPlatform,
executablePathByBrowser,
} from './browsers/browsers.js';
import {detectPlatform} from './detectPlatform.js';
import {detectBrowserPlatform} from './detectPlatform.js';
import os from 'os';
import path from 'path';
import {CacheStructure} from './CacheStructure.js';
/**
* @public
@ -29,7 +31,7 @@ export interface Options {
/**
* Root path to the storage directory.
*/
path: string;
cacheDir: string;
/**
* Determines which platform the browser will be suited for.
*
@ -48,15 +50,19 @@ export interface Options {
}
export function computeExecutablePath(options: Options): string {
options.platform ??= detectPlatform();
options.platform ??= detectBrowserPlatform();
if (!options.platform) {
throw new Error(
`Cannot download a binary for the provided platform: ${os.platform()} (${os.arch()})`
);
}
return executablePathByBrowser[options.browser](
const installationDir = new CacheStructure(options.cacheDir).installationDir(
options.browser,
options.platform,
options.revision,
options.path
options.revision
);
return path.join(
installationDir,
executablePathByBrowser[options.browser](options.platform, options.revision)
);
}

28
packages/browsers/test/src/chrome-data.spec.ts
View File

@ -16,7 +16,7 @@
import {
resolveDownloadUrl,
executablePath,
relativeExecutablePath,
} from '../../lib/cjs/browsers/chrome.js';
import {BrowserPlatform} from '../../lib/cjs/browsers/browsers.js';
import assert from 'assert';
@ -48,30 +48,24 @@ describe('Chrome', () => {
it('should resolve executable paths', () => {
assert.strictEqual(
executablePath(BrowserPlatform.LINUX, '12372323'),
path.join('linux-12372323', 'chrome')
relativeExecutablePath(BrowserPlatform.LINUX, '12372323'),
path.join('chrome')
);
assert.strictEqual(
executablePath(BrowserPlatform.MAC, '12372323'),
path.join('mac-12372323', 'Chromium.app', 'Contents', 'MacOS', 'Chromium')
relativeExecutablePath(BrowserPlatform.MAC, '12372323'),
path.join('Chromium.app', 'Contents', 'MacOS', 'Chromium')
);
assert.strictEqual(
executablePath(BrowserPlatform.MAC_ARM, '12372323'),
path.join(
'mac_arm-12372323',
'Chromium.app',
'Contents',
'MacOS',
'Chromium'
)
relativeExecutablePath(BrowserPlatform.MAC_ARM, '12372323'),
path.join('Chromium.app', 'Contents', 'MacOS', 'Chromium')
);
assert.strictEqual(
executablePath(BrowserPlatform.WIN32, '12372323'),
path.join('win32-12372323', 'chrome.exe')
relativeExecutablePath(BrowserPlatform.WIN32, '12372323'),
path.join('chrome.exe')
);
assert.strictEqual(
executablePath(BrowserPlatform.WIN64, '12372323'),
path.join('win64-12372323', 'chrome.exe')
relativeExecutablePath(BrowserPlatform.WIN64, '12372323'),
path.join('chrome.exe')
);
});
});

15
packages/browsers/test/src/fetch.spec.ts
View File

@ -42,7 +42,7 @@ describe('fetch', () => {
it('should check if a revision can be downloaded', async () => {
assert.ok(
await canFetch({
outputDir: tmpDir,
cacheDir: tmpDir,
browser: Browser.CHROME,
platform: BrowserPlatform.LINUX,
revision: testChromeRevision,
@ -53,7 +53,7 @@ describe('fetch', () => {
it('should report if a revision is not downloadable', async () => {
assert.strictEqual(
await canFetch({
outputDir: tmpDir,
cacheDir: tmpDir,
browser: Browser.CHROME,
platform: BrowserPlatform.LINUX,
revision: 'unknown',
@ -66,11 +66,12 @@ describe('fetch', () => {
this.timeout(60000);
const expectedOutputPath = path.join(
tmpDir,
'chrome',
`${BrowserPlatform.LINUX}-${testChromeRevision}`
);
assert.strictEqual(fs.existsSync(expectedOutputPath), false);
let browser = await fetch({
outputDir: tmpDir,
cacheDir: tmpDir,
browser: Browser.CHROME,
platform: BrowserPlatform.LINUX,
revision: testChromeRevision,
@ -79,7 +80,7 @@ describe('fetch', () => {
assert.ok(fs.existsSync(expectedOutputPath));
// Second iteration should be no-op.
browser = await fetch({
outputDir: tmpDir,
cacheDir: tmpDir,
browser: Browser.CHROME,
platform: BrowserPlatform.LINUX,
revision: testChromeRevision,
@ -92,11 +93,12 @@ describe('fetch', () => {
this.timeout(60000);
const expectedOutputPath = path.join(
tmpDir,
'firefox',
`${BrowserPlatform.LINUX}-${testFirefoxRevision}`
);
assert.strictEqual(fs.existsSync(expectedOutputPath), false);
const browser = await fetch({
outputDir: tmpDir,
cacheDir: tmpDir,
browser: Browser.FIREFOX,
platform: BrowserPlatform.LINUX,
revision: testFirefoxRevision,
@ -113,11 +115,12 @@ describe('fetch', () => {
this.timeout(120000);
const expectedOutputPath = path.join(
tmpDir,
'firefox',
`${BrowserPlatform.MAC}-${testFirefoxRevision}`
);
assert.strictEqual(fs.existsSync(expectedOutputPath), false);
const browser = await fetch({
outputDir: tmpDir,
cacheDir: tmpDir,
browser: Browser.FIREFOX,
platform: BrowserPlatform.MAC,
revision: testFirefoxRevision,

34
packages/browsers/test/src/firefox-data.spec.ts
View File

@ -15,7 +15,7 @@
*/
import {
executablePath,
relativeExecutablePath,
resolveDownloadUrl,
} from '../../lib/cjs/browsers/firefox.js';
import {BrowserPlatform} from '../../lib/cjs/browsers/browsers.js';
@ -48,36 +48,24 @@ describe('Firefox', () => {
it('should resolve executable paths', () => {
assert.strictEqual(
executablePath(BrowserPlatform.LINUX, '111.0a1'),
path.join('linux-111.0a1', 'firefox', 'firefox')
relativeExecutablePath(BrowserPlatform.LINUX, '111.0a1'),
path.join('firefox', 'firefox')
);
assert.strictEqual(
executablePath(BrowserPlatform.MAC, '111.0a1'),
path.join(
'mac-111.0a1',
'Firefox Nightly.app',
'Contents',
'MacOS',
'firefox'
)
relativeExecutablePath(BrowserPlatform.MAC, '111.0a1'),
path.join('Firefox Nightly.app', 'Contents', 'MacOS', 'firefox')
);
assert.strictEqual(
executablePath(BrowserPlatform.MAC_ARM, '111.0a1'),
path.join(
'mac_arm-111.0a1',
'Firefox Nightly.app',
'Contents',
'MacOS',
'firefox'
)
relativeExecutablePath(BrowserPlatform.MAC_ARM, '111.0a1'),
path.join('Firefox Nightly.app', 'Contents', 'MacOS', 'firefox')
);
assert.strictEqual(
executablePath(BrowserPlatform.WIN32, '111.0a1'),
path.join('win32-111.0a1', 'firefox', 'firefox.exe')
relativeExecutablePath(BrowserPlatform.WIN32, '111.0a1'),
path.join('firefox', 'firefox.exe')
);
assert.strictEqual(
executablePath(BrowserPlatform.WIN64, '111.0a1'),
path.join('win64-111.0a1', 'firefox', 'firefox.exe')
relativeExecutablePath(BrowserPlatform.WIN64, '111.0a1'),
path.join('firefox', 'firefox.exe')
);
});
});

8
packages/browsers/test/src/launcher.spec.ts
View File

@ -27,9 +27,9 @@ describe('launcher', () => {
browser: Browser.CHROME,
platform: BrowserPlatform.LINUX,
revision: '123',
path: 'cache',
cacheDir: 'cache',
}),
path.join('cache', 'linux-123', 'chrome')
path.join('cache', 'chrome', 'linux-123', 'chrome')
);
});
it('should compute executable path for Firefox', () => {
@ -38,9 +38,9 @@ describe('launcher', () => {
browser: Browser.FIREFOX,
platform: BrowserPlatform.LINUX,
revision: '123',
path: 'cache',
cacheDir: 'cache',
}),
path.join('cache', 'linux-123', 'firefox', 'firefox')
path.join('cache', 'firefox', 'linux-123', 'firefox', 'firefox')
);
});
});