From 8d5097d7f6c197c3b51df5a69d36083c7f851ef4 Mon Sep 17 00:00:00 2001 From: Alex Rudenko Date: Thu, 15 Sep 2022 07:59:11 +0200 Subject: [PATCH] chore: split CI jobs to run in parallel (#8934) --- .github/workflows/ci.yml | 89 ++++++++++++++++++++++++++++------ package.json | 2 +- test/TestSuites.json | 15 ++++-- utils/generate-matrix.js | 43 ++++++++++++++++ utils/mochaRunner/src/main.ts | 46 +++++++++++------- utils/mochaRunner/src/types.ts | 1 + 6 files changed, 159 insertions(+), 37 deletions(-) create mode 100644 utils/generate-matrix.js diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 51a841f8cae..809e99d0cd5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -122,27 +122,88 @@ jobs: user_email: 55107282+release-please[bot]@users.noreply.github.com tests: - name: ${{ matrix.spec.name }} tests (${{ matrix.spec.node }}) + name: ${{ matrix.spec.name }} tests on ${{ matrix.spec.machine }}(${{ matrix.spec.node }}) runs-on: ${{ matrix.spec.machine }} continue-on-error: true strategy: matrix: spec: - - name: Linux + - name: chrome-headless machine: ubuntu-latest + xvfb: true node: 14 - - name: Linux + suite: chrome-headless + - name: chrome-headless machine: ubuntu-latest + xvfb: true node: 16 - - name: Linux + suite: chrome-headless + - name: chrome-headless machine: ubuntu-latest + xvfb: true node: 18 - - name: macOS - machine: macos-latest - node: 14 - - name: Windows + suite: chrome-headless + - name: chrome-headless machine: windows-latest + xvfb: false + node: 18 + suite: chrome-headless + - name: chrome-headless + machine: macos-latest + xvfb: false + node: 18 + suite: chrome-headless + - name: chrome-headful + machine: ubuntu-latest + xvfb: true node: 14 + suite: chrome-headful + - name: chrome-headful + machine: ubuntu-latest + xvfb: true + node: 16 + suite: chrome-headful + - name: chrome-headful + machine: ubuntu-latest + xvfb: true + node: 18 + suite: chrome-headful + - name: chrome-new-headless + machine: ubuntu-latest + xvfb: true + node: 14 + suite: chrome-new-headless + - name: chrome-new-headless + machine: ubuntu-latest + xvfb: true + node: 16 + suite: chrome-new-headless + - name: chrome-new-headless + machine: ubuntu-latest + xvfb: true + node: 18 + suite: chrome-new-headless + - name: firefox-headless + machine: ubuntu-latest + xvfb: true + node: 14 + suite: firefox-headless + - name: firefox-headless + machine: ubuntu-latest + xvfb: true + node: 16 + suite: firefox-headless + - name: firefox-headless + machine: ubuntu-latest + xvfb: true + node: 18 + suite: firefox-headless + - name: firefox-bidi + machine: ubuntu-latest + xvfb: true + node: 18 + suite: firefox-bidi + steps: - name: Checkout uses: actions/checkout@v3 @@ -167,20 +228,20 @@ jobs: PUPPETEER_PRODUCT: firefox run: npm install - name: Install linux dependencies. - if: ${{ matrix.spec.name == 'Linux' }} + if: ${{ matrix.spec.xvfb }} run: sudo apt-get install xvfb - name: Build run: npm run build:dev - name: Test types run: npm run test:types - name: Run all tests with xvfb - if: ${{ matrix.spec.name == 'Linux' }} - run: xvfb-run --auto-servernum npm run test + if: ${{ matrix.spec.xvfb }} + run: xvfb-run --auto-servernum npm run test -- --test-suite ${{ matrix.spec.suite }} - name: Run all tests without xvfb - if: ${{ matrix.spec.name != 'Linux' }} - run: npm run test + if: ${{ !matrix.spec.xvfb }} + run: npm run test -- --test-suite ${{ matrix.spec.suite }} - name: Test bundling and installation - if: ${{ matrix.spec.name == 'Linux' }} + if: ${{ matrix.spec.machine == 'ubuntu-latest' }} run: | # Test prod build instead of dev. npm run clean diff --git a/package.json b/package.json index 12b497bbb11..12b44d4a381 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,7 @@ "node": ">=14.1.0" }, "scripts": { - "test": "cross-env MOZ_WEBRENDER=0 PUPPETEER_DEFERRED_PROMISE_DEBUG_TIMEOUT=20000 c8 --check-coverage --lines 93 node utils/mochaRunner/lib/main.js", + "test": "cross-env MOZ_WEBRENDER=0 PUPPETEER_DEFERRED_PROMISE_DEBUG_TIMEOUT=20000 node utils/mochaRunner/lib/main.js", "test:types": "tsd", "test:install": "scripts/test-install.sh", "test:firefox": "npm run test -- --test-suite firefox-headless", diff --git a/test/TestSuites.json b/test/TestSuites.json index dfa1cade5dd..d0c208a661d 100644 --- a/test/TestSuites.json +++ b/test/TestSuites.json @@ -3,27 +3,32 @@ { "id": "chrome-headless", "platforms": ["linux", "win32", "darwin"], - "parameters": ["chrome", "headless"] + "parameters": ["chrome", "headless"], + "expectedLineCoverage": 93 }, { "id": "chrome-headful", "platforms": ["linux"], - "parameters": ["chrome", "headful"] + "parameters": ["chrome", "headful"], + "expectedLineCoverage": 93 }, { "id": "chrome-new-headless", "platforms": ["linux"], - "parameters": ["chrome", "chrome-headless"] + "parameters": ["chrome", "chrome-headless"], + "expectedLineCoverage": 93 }, { "id": "firefox-headless", "platforms": ["linux"], - "parameters": ["firefox", "headless"] + "parameters": ["firefox", "headless"], + "expectedLineCoverage": 80 }, { "id": "firefox-bidi", "platforms": ["linux"], - "parameters": ["firefox", "headless", "webDriverBiDi"] + "parameters": ["firefox", "headless", "webDriverBiDi"], + "expectedLineCoverage": 56 } ], "parameterDefinitons": { diff --git a/utils/generate-matrix.js b/utils/generate-matrix.js new file mode 100644 index 00000000000..1cb65d34914 --- /dev/null +++ b/utils/generate-matrix.js @@ -0,0 +1,43 @@ +const fs = require('fs'); + +const data = JSON.parse(fs.readFileSync('./test/TestSuites.json', 'utf-8')); + +/** + * @param {string} platform + * @returns {string} + */ +function mapPlatform(platform) { + switch (platform) { + case 'linux': + return 'ubuntu-latest'; + case 'win32': + return 'windows-latest'; + case 'darwin': + return 'macos-latest'; + default: + throw new Error('Unsupported platform'); + } +} + +const result = []; +for (const suite of data.testSuites) { + for (const platform of suite.platforms) { + if (platform === 'linux' && suite.id !== 'firefox-bidi') { + for (const node of [14, 16, 18]) { + result.push(`- name: ${suite.id} + machine: ${mapPlatform(platform)} + xvfb: true + node: ${node} + suite: ${suite.id}`); + } + } else { + result.push(`- name: ${suite.id} + machine: ${mapPlatform(platform)} + xvfb: ${platform === 'linux'} + node: 18 + suite: ${suite.id}`); + } + } +} + +console.log(result.join('\n')); diff --git a/utils/mochaRunner/src/main.ts b/utils/mochaRunner/src/main.ts index 24758a00746..65fb13e8f4e 100644 --- a/utils/mochaRunner/src/main.ts +++ b/utils/mochaRunner/src/main.ts @@ -70,6 +70,8 @@ function getApplicableTestSuites( } async function main() { + const noCoverage = process.argv.indexOf('--no-coverage') !== -1; + const platform = zPlatform.parse(os.platform()); const expectations = readJSON( @@ -116,23 +118,33 @@ async function main() { ); const tmpFilename = path.join(tmpDir, 'output.json'); console.log('Running', JSON.stringify(parameters), tmpFilename); - const handle = spawn( - 'npx mocha', - [ - '-u', - path.join(__dirname, 'interface.js'), - '-R', - path.join(__dirname, 'reporter.js'), - '-O', - 'output=' + tmpFilename, - ], - { - shell: true, - cwd: process.cwd(), - stdio: 'inherit', - env, - } - ); + const args = [ + '-u', + path.join(__dirname, 'interface.js'), + '-R', + path.join(__dirname, 'reporter.js'), + '-O', + 'output=' + tmpFilename, + ]; + const handle = noCoverage + ? spawn('npx', ['mocha', ...args]) + : spawn( + 'npx', + [ + 'c8', + '--check-coverage', + '--lines', + String(suite.expectedLineCoverage), + 'npx mocha', + ...args, + ], + { + shell: true, + cwd: process.cwd(), + stdio: 'inherit', + env, + } + ); await new Promise((resolve, reject) => { handle.on('error', err => { reject(err); diff --git a/utils/mochaRunner/src/types.ts b/utils/mochaRunner/src/types.ts index 31fea50916f..e8fac25afc3 100644 --- a/utils/mochaRunner/src/types.ts +++ b/utils/mochaRunner/src/types.ts @@ -24,6 +24,7 @@ export const zTestSuite = z.object({ id: z.string(), platforms: z.array(zPlatform), parameters: z.array(z.string()), + expectedLineCoverage: z.number(), }); export type TestSuite = z.infer;