chore: split CI jobs to run in parallel (#8934)

.github/workflows/ci.yml

@@ -122,27 +122,88 @@ jobs:
           user_email: 55107282+release-please[bot]@users.noreply.github.com
 
   tests:
-    name: ${{ matrix.spec.name }} tests (${{ matrix.spec.node }})
+    name: ${{ matrix.spec.name }} tests on ${{ matrix.spec.machine }}(${{ matrix.spec.node }})
     runs-on: ${{ matrix.spec.machine }}
     continue-on-error: true
     strategy:
       matrix:
         spec:
-          - name: Linux
+          - name: chrome-headless
             machine: ubuntu-latest
+            xvfb: true
             node: 14
-          - name: Linux
+            suite: chrome-headless
+          - name: chrome-headless
             machine: ubuntu-latest
+            xvfb: true
             node: 16
-          - name: Linux
+            suite: chrome-headless
+          - name: chrome-headless
             machine: ubuntu-latest
+            xvfb: true
             node: 18
-          - name: macOS
+            suite: chrome-headless
-            machine: macos-latest
+          - name: chrome-headless
-            node: 14
-          - name: Windows
             machine: windows-latest
+            xvfb: false
+            node: 18
+            suite: chrome-headless
+          - name: chrome-headless
+            machine: macos-latest
+            xvfb: false
+            node: 18
+            suite: chrome-headless
+          - name: chrome-headful
+            machine: ubuntu-latest
+            xvfb: true
             node: 14
+            suite: chrome-headful
+          - name: chrome-headful
+            machine: ubuntu-latest
+            xvfb: true
+            node: 16
+            suite: chrome-headful
+          - name: chrome-headful
+            machine: ubuntu-latest
+            xvfb: true
+            node: 18
+            suite: chrome-headful
+          - name: chrome-new-headless
+            machine: ubuntu-latest
+            xvfb: true
+            node: 14
+            suite: chrome-new-headless
+          - name: chrome-new-headless
+            machine: ubuntu-latest
+            xvfb: true
+            node: 16
+            suite: chrome-new-headless
+          - name: chrome-new-headless
+            machine: ubuntu-latest
+            xvfb: true
+            node: 18
+            suite: chrome-new-headless
+          - name: firefox-headless
+            machine: ubuntu-latest
+            xvfb: true
+            node: 14
+            suite: firefox-headless
+          - name: firefox-headless
+            machine: ubuntu-latest
+            xvfb: true
+            node: 16
+            suite: firefox-headless
+          - name: firefox-headless
+            machine: ubuntu-latest
+            xvfb: true
+            node: 18
+            suite: firefox-headless
+          - name: firefox-bidi
+            machine: ubuntu-latest
+            xvfb: true
+            node: 18
+            suite: firefox-bidi
+
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -167,20 +228,20 @@ jobs:
           PUPPETEER_PRODUCT: firefox
         run: npm install
       - name: Install linux dependencies.
-        if: ${{ matrix.spec.name == 'Linux' }}
+        if: ${{ matrix.spec.xvfb }}
         run: sudo apt-get install xvfb
       - name: Build
         run: npm run build:dev
       - name: Test types
         run: npm run test:types
       - name: Run all tests with xvfb
-        if: ${{ matrix.spec.name == 'Linux' }}
+        if: ${{ matrix.spec.xvfb }}
-        run: xvfb-run --auto-servernum npm run test
+        run: xvfb-run --auto-servernum npm run test -- --test-suite ${{ matrix.spec.suite }}
       - name: Run all tests without xvfb
-        if: ${{ matrix.spec.name != 'Linux' }}
+        if: ${{ !matrix.spec.xvfb }}
-        run: npm run test
+        run: npm run test -- --test-suite ${{ matrix.spec.suite }}
       - name: Test bundling and installation
-        if: ${{ matrix.spec.name == 'Linux' }}
+        if: ${{ matrix.spec.machine == 'ubuntu-latest' }}
         run: |
           # Test prod build instead of dev.
           npm run clean

package.json

@@ -27,7 +27,7 @@
     "node": ">=14.1.0"
   },
   "scripts": {
-    "test": "cross-env MOZ_WEBRENDER=0 PUPPETEER_DEFERRED_PROMISE_DEBUG_TIMEOUT=20000 c8 --check-coverage --lines 93 node utils/mochaRunner/lib/main.js",
+    "test": "cross-env MOZ_WEBRENDER=0 PUPPETEER_DEFERRED_PROMISE_DEBUG_TIMEOUT=20000 node utils/mochaRunner/lib/main.js",
     "test:types": "tsd",
     "test:install": "scripts/test-install.sh",
     "test:firefox": "npm run test -- --test-suite firefox-headless",

test/TestSuites.json

@@ -3,27 +3,32 @@
     {
       "id": "chrome-headless",
       "platforms": ["linux", "win32", "darwin"],
-      "parameters": ["chrome", "headless"]
+      "parameters": ["chrome", "headless"],
+      "expectedLineCoverage": 93
     },
     {
       "id": "chrome-headful",
       "platforms": ["linux"],
-      "parameters": ["chrome", "headful"]
+      "parameters": ["chrome", "headful"],
+      "expectedLineCoverage": 93
     },
     {
       "id": "chrome-new-headless",
       "platforms": ["linux"],
-      "parameters": ["chrome", "chrome-headless"]
+      "parameters": ["chrome", "chrome-headless"],
+      "expectedLineCoverage": 93
     },
     {
       "id": "firefox-headless",
       "platforms": ["linux"],
-      "parameters": ["firefox", "headless"]
+      "parameters": ["firefox", "headless"],
+      "expectedLineCoverage": 80
     },
     {
       "id": "firefox-bidi",
       "platforms": ["linux"],
-      "parameters": ["firefox", "headless", "webDriverBiDi"]
+      "parameters": ["firefox", "headless", "webDriverBiDi"],
+      "expectedLineCoverage": 56
     }
   ],
   "parameterDefinitons": {

utils/generate-matrix.js

@@ -0,0 +1,43 @@
+const fs = require('fs');
+
+const data = JSON.parse(fs.readFileSync('./test/TestSuites.json', 'utf-8'));
+
+/**
+ * @param {string} platform
+ * @returns {string}
+ */
+function mapPlatform(platform) {
+  switch (platform) {
+    case 'linux':
+      return 'ubuntu-latest';
+    case 'win32':
+      return 'windows-latest';
+    case 'darwin':
+      return 'macos-latest';
+    default:
+      throw new Error('Unsupported platform');
+  }
+}
+
+const result = [];
+for (const suite of data.testSuites) {
+  for (const platform of suite.platforms) {
+    if (platform === 'linux' && suite.id !== 'firefox-bidi') {
+      for (const node of [14, 16, 18]) {
+        result.push(`- name: ${suite.id}
+  machine: ${mapPlatform(platform)}
+  xvfb: true
+  node: ${node}
+  suite: ${suite.id}`);
+      }
+    } else {
+      result.push(`- name: ${suite.id}
+  machine: ${mapPlatform(platform)}
+  xvfb: ${platform === 'linux'}
+  node: 18
+  suite: ${suite.id}`);
+    }
+  }
+}
+
+console.log(result.join('\n'));

utils/mochaRunner/src/main.ts

@@ -70,6 +70,8 @@ function getApplicableTestSuites(
 }
 
 async function main() {
+  const noCoverage = process.argv.indexOf('--no-coverage') !== -1;
+
   const platform = zPlatform.parse(os.platform());
 
   const expectations = readJSON(
@@ -116,23 +118,33 @@ async function main() {
       );
       const tmpFilename = path.join(tmpDir, 'output.json');
       console.log('Running', JSON.stringify(parameters), tmpFilename);
-      const handle = spawn(
+      const args = [
-        'npx mocha',
+        '-u',
-        [
+        path.join(__dirname, 'interface.js'),
-          '-u',
+        '-R',
-          path.join(__dirname, 'interface.js'),
+        path.join(__dirname, 'reporter.js'),
-          '-R',
+        '-O',
-          path.join(__dirname, 'reporter.js'),
+        'output=' + tmpFilename,
-          '-O',
+      ];
-          'output=' + tmpFilename,
+      const handle = noCoverage
-        ],
+        ? spawn('npx', ['mocha', ...args])
-        {
+        : spawn(
-          shell: true,
+            'npx',
-          cwd: process.cwd(),
+            [
-          stdio: 'inherit',
+              'c8',
-          env,
+              '--check-coverage',
-        }
+              '--lines',
-      );
+              String(suite.expectedLineCoverage),
+              'npx mocha',
+              ...args,
+            ],
+            {
+              shell: true,
+              cwd: process.cwd(),
+              stdio: 'inherit',
+              env,
+            }
+          );
       await new Promise<void>((resolve, reject) => {
         handle.on('error', err => {
           reject(err);

utils/mochaRunner/src/types.ts

@@ -24,6 +24,7 @@ export const zTestSuite = z.object({
   id: z.string(),
   platforms: z.array(zPlatform),
   parameters: z.array(z.string()),
+  expectedLineCoverage: z.number(),
 });
 
 export type TestSuite = z.infer<typeof zTestSuite>;