chore: follow test:$browser[:$flavor] naming for tests + retries for Firefox (#8775)

.github/workflows/ci.yml

@@ -168,16 +168,24 @@ jobs:
         run: npm run build
       - name: Test types
         run: npm run test:types
-      - name: Run all tests (only on Linux)
-        id: full-test
+      # On Linux we run all Chrome tests without retries and Firefox tests with retries.
+      - name: Run all Chrome tests (only on Linux)
         if: ${{ matrix.spec.name == 'Linux' }}
-        run: xvfb-run --auto-servernum npm test
-      - name: Test Chrome
+        run: xvfb-run --auto-servernum npm run test:chrome
+      - name: Run all Firefox tests (only on Linux)
+        if: ${{ matrix.spec.name == 'Linux' }}
+        uses: nick-invision/retry@v2
+        with:
+          command: xvfb-run --auto-servernum npm run test:firefox
+          timeout_minutes: 20
+          max_attempts: 3
+      # On other platforms we only run chrome:headless tests and continue on Firefox errors.
+      - name: Test Chrome Headless (not on Linux)
         id: test-chrome
-        if: ${{ steps.full-test.conclusion == 'skipped' }}
-        run: npm run test:chrome
-      - name: Test Firefox
-        if: ${{ steps.test-chrome.conclusion != 'skipped' }}
+        if: ${{ matrix.spec.name != 'Linux' }}
+        run: npm run test:chrome:headless
+      - name: Run all Firefox tests (not on Linux)
+        if: ${{ matrix.spec.name != 'Linux' }}
         continue-on-error: true
         run: npm run test:firefox
       - name: Test bundling and installation

.github/workflows/tot-ci.yml

@@ -49,7 +49,7 @@ jobs:
       - name: Run unit tests in headless
         uses: nick-invision/retry@v2
         with:
-          command: xvfb-run --auto-servernum npm run test:chrome
+          command: xvfb-run --auto-servernum npm run test:chrome:headless
           timeout_minutes: 10
           max_attempts: 3
 
@@ -125,6 +125,6 @@ jobs:
       - name: Run unit tests in chrome headless
         uses: nick-invision/retry@v2
         with:
-          command: xvfb-run --auto-servernum npm run test:chrome:headless
+          command: xvfb-run --auto-servernum npm run test:chrome:headless-chrome
           timeout_minutes: 10
           max_attempts: 3

docs/contributing.md

@@ -208,7 +208,7 @@ npm run test
 - To run Chrome tests in non-headless mode:
 
 ```bash
-HEADLESS=false npm run test:chrome
+HEADLESS=false npm run test:chrome:headless
 ```
 
 - To run Firefox tests, firstly ensure you have Firefox installed locally (you only need to do this once, not on every test run) and then you can run the tests:
@@ -222,13 +222,13 @@ npm run test:firefox
 
 ```bash
 PUPPETEER_EXPERIMENTAL_CHROMIUM_MAC_ARM=1 npm install
-PUPPETEER_EXPERIMENTAL_CHROMIUM_MAC_ARM=1 npm run test:chrome
+PUPPETEER_EXPERIMENTAL_CHROMIUM_MAC_ARM=1 npm run test:chrome:headless
 ```
 
 - To run tests with custom browser executable:
 
 ```bash
-BINARY=<path-to-executable> npm run test:chrome # Or npm run test:firefox
+BINARY=<path-to-executable> npm run test:chrome:headless # Or npm run test:firefox
 ```
 
 ## API Coverage

package.json

@@ -27,12 +27,13 @@
     "node": ">=14.1.0"
   },
   "scripts": {
-    "test": "c8 --check-coverage --lines 93 run-s test:chrome test:chrome:* test:firefox",
+    "test": "c8 --check-coverage --lines 93 run-s test:chrome:* test:firefox",
     "test:types": "tsd",
     "test:install": "scripts/test-install.sh",
     "test:firefox": "cross-env PUPPETEER_PRODUCT=firefox MOZ_WEBRENDER=0 mocha",
-    "test:chrome": "mocha",
-    "test:chrome:headless": "cross-env HEADLESS=chrome mocha",
+    "test:chrome": "run-s test:chrome:*",
+    "test:chrome:headless": "cross-env HEADLESS=true mocha",
+    "test:chrome:headless-chrome": "cross-env HEADLESS=chrome mocha",
     "test:chrome:headful": "cross-env HEADLESS=false mocha",
     "prepublishOnly": "npm run build",
     "prepare": "node typescript-if-required.js && husky install",

test/README.md

@@ -81,5 +81,5 @@ npm run test:chrome:headful
 - To run tests with custom browser executable:
 
 ```bash
-BINARY=<path-to-executable> npm run test:chrome # Or npm run test:firefox
+BINARY=<path-to-executable> npm run test:chrome:headless # Or npm run test:firefox
 ```

utils/bisect.js

@@ -35,7 +35,7 @@ Usage:
   node bisect.js --good <revision> --bad <revision> <script>
 
 Parameters:
-  --good       revision that is known to be GOOD, defaults to the chromium revision in src/revision.ts in the main branch 
+  --good       revision that is known to be GOOD, defaults to the chromium revision in src/revision.ts in the main branch
   --bad        revision that is known to be BAD, defaults to the chromium revision in src/revision.ts
   --no-cache   do not keep downloaded Chromium revisions
   --unit-test  pattern that identifies a unit tests that should be checked
@@ -180,7 +180,7 @@ function runScript(scriptPath, revisionInfo) {
 function runUnitTest(pattern, revisionInfo) {
   const log = debug('bisect:rununittest');
   log('Running unit test');
-  const child = spawn('npm run test:chrome', ['--', '-g', pattern], {
+  const child = spawn('npm run test:chrome:headless', ['--', '-g', pattern], {
     stdio: ['inherit', 'inherit', 'inherit', 'ipc'],
     shell: true,
     env: {