test: test sharding for firefox (#11008)

Co-authored-by: jrandolf <101637635+jrandolf@users.noreply.github.com>

.github/workflows/ci.yml

@@ -220,7 +220,7 @@ jobs:
       - run: 'exit 0'
 
   firefox-tests:
-    name: ${{ matrix.suite }} tests on ${{ matrix.os }}
+    name: ${{ matrix.suite }} tests on ${{ matrix.os }} (${{ matrix.shard }})
     runs-on: ${{ matrix.os }}
     needs: check-changes
     if: ${{ contains(fromJSON(needs.check-changes.outputs.changes), 'puppeteer') }}
@@ -234,6 +234,11 @@ jobs:
           - firefox-bidi
           - firefox-headful
           - firefox-headless
+        shard:
+          - 1/4
+          - 2/4
+          - 3/4
+          - 4/4
         exclude:
           - os: macos-latest
             suite: firefox-headful
@@ -268,10 +273,10 @@ jobs:
         run: npm run postinstall
       - name: Run all tests (for non-Linux)
         if: ${{ matrix.os != 'ubuntu-latest' }}
-        run: npm run test -- --test-suite ${{ matrix.suite }} --save-stats-to /tmp/artifacts/${{ github.event_name }}_INSERTID.json
+        run: npm run test -- --shard '${{ matrix.shard }}' --test-suite ${{ matrix.suite }} --save-stats-to /tmp/artifacts/${{ github.event_name }}_INSERTID.json
       - name: Run all tests (for Linux)
         if: ${{ matrix.os == 'ubuntu-latest' }}
-        run: xvfb-run --auto-servernum npm run test -- --test-suite ${{ matrix.suite }} --save-stats-to /tmp/artifacts/${{ github.event_name }}_INSERTID.json
+        run: xvfb-run --auto-servernum npm run test -- --shard '${{ matrix.shard }}' --test-suite ${{ matrix.suite }} --save-stats-to /tmp/artifacts/${{ github.event_name }}_INSERTID.json
       - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
         if: always()
         with:

.mocharc.cjs

@@ -22,10 +22,14 @@ module.exports = {
   reporter: 'dot',
   logLevel: 'debug',
   require: ['./test/build/mocha-utils.js', 'source-map-support/register'],
-  spec: 'test/build/**/*.spec.js',
   exit: !!process.env.CI,
   retries: process.env.CI ? 3 : 0,
   parallel: !!process.env.PARALLEL,
   timeout: timeout,
   reporter: process.env.CI ? 'spec' : 'dot',
+  ...(!process.env['PUPPETEER_SHARD']
+    ? {
+        spec: 'test/build/**/*.spec.js',
+      }
+    : {}),
 };

test/TestExpectations.json

@@ -1481,12 +1481,6 @@
     "parameters": ["cdp", "firefox"],
     "expectations": ["FAIL", "PASS"]
   },
-  {
-    "testIdPattern": "[click.spec] Page.click should click the button after navigation",
-    "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["firefox", "webDriverBiDi"],
-    "expectations": ["FAIL"]
-  },
   {
     "testIdPattern": "[click.spec] Page.click should click the button if window.Node is removed",
     "platforms": ["darwin", "linux", "win32"],
@@ -1637,12 +1631,6 @@
     "parameters": ["chrome", "webDriverBiDi"],
     "expectations": ["FAIL", "PASS"]
   },
-  {
-    "testIdPattern": "[defaultbrowsercontext.spec] DefaultBrowserContext page.cookies() should work",
-    "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["cdp", "firefox"],
-    "expectations": ["FAIL"]
-  },
   {
     "testIdPattern": "[defaultbrowsercontext.spec] DefaultBrowserContext page.deleteCookie() should work",
     "platforms": ["darwin", "linux", "win32"],
@@ -2331,7 +2319,7 @@
     "testIdPattern": "[navigation.spec] navigation Frame.waitForNavigation should work",
     "platforms": ["darwin", "linux", "win32"],
     "parameters": ["firefox", "webDriverBiDi"],
-    "expectations": ["TIMEOUT"]
+    "expectations": ["FAIL", "TIMEOUT"]
   },
   {
     "testIdPattern": "[navigation.spec] navigation Page.goBack should work with HistoryAPI",
@@ -2471,12 +2459,6 @@
     "parameters": ["cdp", "firefox"],
     "expectations": ["SKIP"]
   },
-  {
-    "testIdPattern": "[navigation.spec] navigation Page.waitForNavigation should work",
-    "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["firefox", "webDriverBiDi"],
-    "expectations": ["TIMEOUT"]
-  },
   {
     "testIdPattern": "[navigation.spec] navigation Page.waitForNavigation should work when subframe issues window.stop()",
     "platforms": ["darwin", "linux", "win32"],
@@ -2495,12 +2477,6 @@
     "parameters": ["chrome", "webDriverBiDi"],
     "expectations": ["FAIL"]
   },
-  {
-    "testIdPattern": "[navigation.spec] navigation Page.waitForNavigation should work with both domcontentloaded and load",
-    "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["firefox", "webDriverBiDi"],
-    "expectations": ["TIMEOUT"]
-  },
   {
     "testIdPattern": "[navigation.spec] navigation Page.waitForNavigation should work with clicking on anchor links",
     "platforms": ["darwin", "linux", "win32"],
@@ -3603,7 +3579,7 @@
     "testIdPattern": "[target.spec] Target should not report uninitialized pages",
     "platforms": ["darwin", "linux", "win32"],
     "parameters": ["firefox", "webDriverBiDi"],
-    "expectations": ["PASS"]
+    "expectations": ["FAIL", "PASS"]
   },
   {
     "testIdPattern": "[target.spec] Target should report when a new page is created and closed",

tools/mocha-runner/src/mocha-runner.ts

@@ -22,6 +22,8 @@ import {spawn, type SpawnOptions} from 'node:child_process';
 import os from 'os';
 import path from 'path';
 
+import {globSync} from 'glob';
+
 import {
   zPlatform,
   zTestSuiteFile,
@@ -93,6 +95,12 @@ async function main() {
     minTests = Number(process.argv[minTestsIdx + 1]);
   }
 
+  const shardIdx = process.argv.indexOf('--shard');
+  let shard = null;
+  if (shardIdx !== -1) {
+    shard = String(process.argv[shardIdx + 1]);
+  }
+
   const platform = zPlatform.parse(os.platform());
 
   const expectations = readJSON(
@@ -182,13 +190,41 @@ async function main() {
         args.push('--no-parallel');
       }
       if (process.argv.indexOf('--fullTrace')) {
-        args.push('--fullTrace');
+        args.push('--full-trace');
+      }
+      if (shard) {
+        const specs = globSync('test/build/**/*.spec.js').sort((a, b) => {
+          return a.localeCompare(b);
+        });
+        // Shard ID is 1-based.
+        const [shardId, shards] = shard.split('/').map(s => {
+          return Number(s);
+        }) as [number, number];
+        const argsLength = args.length;
+        for (let i = 0; i < specs.length; i++) {
+          if (i % shards === shardId - 1) {
+            args.push(specs[i]!);
+          }
+        }
+        if (argsLength === args.length) {
+          throw new Error('Shard did not result in any test files');
+        }
+        console.log(
+          `Running shard ${shardId}/${shards}. Picked ${
+            args.length - argsLength
+          } files out of ${specs.length}.`
+        );
       }
       const spawnArgs: SpawnOptions = {
         shell: true,
         cwd: process.cwd(),
         stdio: 'inherit',
-        env,
+        env: shard
+          ? {
+              ...env,
+              PUPPETEER_SHARD: 'true',
+            }
+          : env,
       };
       const handle = noCoverage
         ? spawn('npx', ['mocha', ...args], spawnArgs)
@@ -229,7 +265,7 @@ async function main() {
           results.updates = updates;
           writeJSON(tmpFilename, results);
         } else {
-          if (totalTests < minTests) {
+          if (!shard && totalTests < minTests) {
             fail = true;
             console.log(
               `Test run matches expectations but the number of discovered tests is too low (expected: ${minTests}, actual: ${totalTests}).`