From 52468adcff58190a9000bb4264ee0351502b3e72 Mon Sep 17 00:00:00 2001 From: Alex Rudenko Date: Mon, 25 Sep 2023 14:00:09 +0200 Subject: [PATCH] test: test sharding for firefox (#11008) Co-authored-by: jrandolf <101637635+jrandolf@users.noreply.github.com> --- .github/workflows/ci.yml | 11 +++++-- .mocharc.cjs | 6 +++- test/TestExpectations.json | 28 ++--------------- tools/mocha-runner/src/mocha-runner.ts | 42 ++++++++++++++++++++++++-- 4 files changed, 54 insertions(+), 33 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 909082a9ea2..69827164cff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -220,7 +220,7 @@ jobs: - run: 'exit 0' firefox-tests: - name: ${{ matrix.suite }} tests on ${{ matrix.os }} + name: ${{ matrix.suite }} tests on ${{ matrix.os }} (${{ matrix.shard }}) runs-on: ${{ matrix.os }} needs: check-changes if: ${{ contains(fromJSON(needs.check-changes.outputs.changes), 'puppeteer') }} @@ -234,6 +234,11 @@ jobs: - firefox-bidi - firefox-headful - firefox-headless + shard: + - 1/4 + - 2/4 + - 3/4 + - 4/4 exclude: - os: macos-latest suite: firefox-headful @@ -268,10 +273,10 @@ jobs: run: npm run postinstall - name: Run all tests (for non-Linux) if: ${{ matrix.os != 'ubuntu-latest' }} - run: npm run test -- --test-suite ${{ matrix.suite }} --save-stats-to /tmp/artifacts/${{ github.event_name }}_INSERTID.json + run: npm run test -- --shard '${{ matrix.shard }}' --test-suite ${{ matrix.suite }} --save-stats-to /tmp/artifacts/${{ github.event_name }}_INSERTID.json - name: Run all tests (for Linux) if: ${{ matrix.os == 'ubuntu-latest' }} - run: xvfb-run --auto-servernum npm run test -- --test-suite ${{ matrix.suite }} --save-stats-to /tmp/artifacts/${{ github.event_name }}_INSERTID.json + run: xvfb-run --auto-servernum npm run test -- --shard '${{ matrix.shard }}' --test-suite ${{ matrix.suite }} --save-stats-to /tmp/artifacts/${{ github.event_name }}_INSERTID.json - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 if: always() with: diff --git a/.mocharc.cjs b/.mocharc.cjs index 8414501ced2..fff31ff0021 100644 --- a/.mocharc.cjs +++ b/.mocharc.cjs @@ -22,10 +22,14 @@ module.exports = { reporter: 'dot', logLevel: 'debug', require: ['./test/build/mocha-utils.js', 'source-map-support/register'], - spec: 'test/build/**/*.spec.js', exit: !!process.env.CI, retries: process.env.CI ? 3 : 0, parallel: !!process.env.PARALLEL, timeout: timeout, reporter: process.env.CI ? 'spec' : 'dot', + ...(!process.env['PUPPETEER_SHARD'] + ? { + spec: 'test/build/**/*.spec.js', + } + : {}), }; diff --git a/test/TestExpectations.json b/test/TestExpectations.json index e96be62e883..eb74f7db693 100644 --- a/test/TestExpectations.json +++ b/test/TestExpectations.json @@ -1481,12 +1481,6 @@ "parameters": ["cdp", "firefox"], "expectations": ["FAIL", "PASS"] }, - { - "testIdPattern": "[click.spec] Page.click should click the button after navigation", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["firefox", "webDriverBiDi"], - "expectations": ["FAIL"] - }, { "testIdPattern": "[click.spec] Page.click should click the button if window.Node is removed", "platforms": ["darwin", "linux", "win32"], @@ -1637,12 +1631,6 @@ "parameters": ["chrome", "webDriverBiDi"], "expectations": ["FAIL", "PASS"] }, - { - "testIdPattern": "[defaultbrowsercontext.spec] DefaultBrowserContext page.cookies() should work", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["cdp", "firefox"], - "expectations": ["FAIL"] - }, { "testIdPattern": "[defaultbrowsercontext.spec] DefaultBrowserContext page.deleteCookie() should work", "platforms": ["darwin", "linux", "win32"], @@ -2331,7 +2319,7 @@ "testIdPattern": "[navigation.spec] navigation Frame.waitForNavigation should work", "platforms": ["darwin", "linux", "win32"], "parameters": ["firefox", "webDriverBiDi"], - "expectations": ["TIMEOUT"] + "expectations": ["FAIL", "TIMEOUT"] }, { "testIdPattern": "[navigation.spec] navigation Page.goBack should work with HistoryAPI", @@ -2471,12 +2459,6 @@ "parameters": ["cdp", "firefox"], "expectations": ["SKIP"] }, - { - "testIdPattern": "[navigation.spec] navigation Page.waitForNavigation should work", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["firefox", "webDriverBiDi"], - "expectations": ["TIMEOUT"] - }, { "testIdPattern": "[navigation.spec] navigation Page.waitForNavigation should work when subframe issues window.stop()", "platforms": ["darwin", "linux", "win32"], @@ -2495,12 +2477,6 @@ "parameters": ["chrome", "webDriverBiDi"], "expectations": ["FAIL"] }, - { - "testIdPattern": "[navigation.spec] navigation Page.waitForNavigation should work with both domcontentloaded and load", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["firefox", "webDriverBiDi"], - "expectations": ["TIMEOUT"] - }, { "testIdPattern": "[navigation.spec] navigation Page.waitForNavigation should work with clicking on anchor links", "platforms": ["darwin", "linux", "win32"], @@ -3603,7 +3579,7 @@ "testIdPattern": "[target.spec] Target should not report uninitialized pages", "platforms": ["darwin", "linux", "win32"], "parameters": ["firefox", "webDriverBiDi"], - "expectations": ["PASS"] + "expectations": ["FAIL", "PASS"] }, { "testIdPattern": "[target.spec] Target should report when a new page is created and closed", diff --git a/tools/mocha-runner/src/mocha-runner.ts b/tools/mocha-runner/src/mocha-runner.ts index 5785f9aa75d..a359ef81e57 100644 --- a/tools/mocha-runner/src/mocha-runner.ts +++ b/tools/mocha-runner/src/mocha-runner.ts @@ -22,6 +22,8 @@ import {spawn, type SpawnOptions} from 'node:child_process'; import os from 'os'; import path from 'path'; +import {globSync} from 'glob'; + import { zPlatform, zTestSuiteFile, @@ -93,6 +95,12 @@ async function main() { minTests = Number(process.argv[minTestsIdx + 1]); } + const shardIdx = process.argv.indexOf('--shard'); + let shard = null; + if (shardIdx !== -1) { + shard = String(process.argv[shardIdx + 1]); + } + const platform = zPlatform.parse(os.platform()); const expectations = readJSON( @@ -182,13 +190,41 @@ async function main() { args.push('--no-parallel'); } if (process.argv.indexOf('--fullTrace')) { - args.push('--fullTrace'); + args.push('--full-trace'); + } + if (shard) { + const specs = globSync('test/build/**/*.spec.js').sort((a, b) => { + return a.localeCompare(b); + }); + // Shard ID is 1-based. + const [shardId, shards] = shard.split('/').map(s => { + return Number(s); + }) as [number, number]; + const argsLength = args.length; + for (let i = 0; i < specs.length; i++) { + if (i % shards === shardId - 1) { + args.push(specs[i]!); + } + } + if (argsLength === args.length) { + throw new Error('Shard did not result in any test files'); + } + console.log( + `Running shard ${shardId}/${shards}. Picked ${ + args.length - argsLength + } files out of ${specs.length}.` + ); } const spawnArgs: SpawnOptions = { shell: true, cwd: process.cwd(), stdio: 'inherit', - env, + env: shard + ? { + ...env, + PUPPETEER_SHARD: 'true', + } + : env, }; const handle = noCoverage ? spawn('npx', ['mocha', ...args], spawnArgs) @@ -229,7 +265,7 @@ async function main() { results.updates = updates; writeJSON(tmpFilename, results); } else { - if (totalTests < minTests) { + if (!shard && totalTests < minTests) { fail = true; console.log( `Test run matches expectations but the number of discovered tests is too low (expected: ${minTests}, actual: ${totalTests}).`