diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 06d0e17d60d..cb55069b09c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -168,16 +168,24 @@ jobs: run: npm run build - name: Test types run: npm run test:types - - name: Run all tests (only on Linux) - id: full-test + # On Linux we run all Chrome tests without retries and Firefox tests with retries. + - name: Run all Chrome tests (only on Linux) if: ${{ matrix.spec.name == 'Linux' }} - run: xvfb-run --auto-servernum npm test - - name: Test Chrome + run: xvfb-run --auto-servernum npm run test:chrome + - name: Run all Firefox tests (only on Linux) + if: ${{ matrix.spec.name == 'Linux' }} + uses: nick-invision/retry@v2 + with: + command: xvfb-run --auto-servernum npm run test:firefox + timeout_minutes: 20 + max_attempts: 3 + # On other platforms we only run chrome:headless tests and continue on Firefox errors. + - name: Test Chrome Headless (not on Linux) id: test-chrome - if: ${{ steps.full-test.conclusion == 'skipped' }} - run: npm run test:chrome - - name: Test Firefox - if: ${{ steps.test-chrome.conclusion != 'skipped' }} + if: ${{ matrix.spec.name != 'Linux' }} + run: npm run test:chrome:headless + - name: Run all Firefox tests (not on Linux) + if: ${{ matrix.spec.name != 'Linux' }} continue-on-error: true run: npm run test:firefox - name: Test bundling and installation diff --git a/.github/workflows/tot-ci.yml b/.github/workflows/tot-ci.yml index aa197843e52..17e72acb390 100644 --- a/.github/workflows/tot-ci.yml +++ b/.github/workflows/tot-ci.yml @@ -49,7 +49,7 @@ jobs: - name: Run unit tests in headless uses: nick-invision/retry@v2 with: - command: xvfb-run --auto-servernum npm run test:chrome + command: xvfb-run --auto-servernum npm run test:chrome:headless timeout_minutes: 10 max_attempts: 3 @@ -125,6 +125,6 @@ jobs: - name: Run unit tests in chrome headless uses: nick-invision/retry@v2 with: - command: xvfb-run --auto-servernum npm run test:chrome:headless + command: xvfb-run --auto-servernum npm run test:chrome:headless-chrome timeout_minutes: 10 max_attempts: 3 diff --git a/docs/contributing.md b/docs/contributing.md index 27a72699115..f991937e514 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -208,7 +208,7 @@ npm run test - To run Chrome tests in non-headless mode: ```bash -HEADLESS=false npm run test:chrome +HEADLESS=false npm run test:chrome:headless ``` - To run Firefox tests, firstly ensure you have Firefox installed locally (you only need to do this once, not on every test run) and then you can run the tests: @@ -222,13 +222,13 @@ npm run test:firefox ```bash PUPPETEER_EXPERIMENTAL_CHROMIUM_MAC_ARM=1 npm install -PUPPETEER_EXPERIMENTAL_CHROMIUM_MAC_ARM=1 npm run test:chrome +PUPPETEER_EXPERIMENTAL_CHROMIUM_MAC_ARM=1 npm run test:chrome:headless ``` - To run tests with custom browser executable: ```bash -BINARY= npm run test:chrome # Or npm run test:firefox +BINARY= npm run test:chrome:headless # Or npm run test:firefox ``` ## API Coverage diff --git a/package.json b/package.json index e41f727edc8..b1b37b22382 100644 --- a/package.json +++ b/package.json @@ -27,12 +27,13 @@ "node": ">=14.1.0" }, "scripts": { - "test": "c8 --check-coverage --lines 93 run-s test:chrome test:chrome:* test:firefox", + "test": "c8 --check-coverage --lines 93 run-s test:chrome:* test:firefox", "test:types": "tsd", "test:install": "scripts/test-install.sh", "test:firefox": "cross-env PUPPETEER_PRODUCT=firefox MOZ_WEBRENDER=0 mocha", - "test:chrome": "mocha", - "test:chrome:headless": "cross-env HEADLESS=chrome mocha", + "test:chrome": "run-s test:chrome:*", + "test:chrome:headless": "cross-env HEADLESS=true mocha", + "test:chrome:headless-chrome": "cross-env HEADLESS=chrome mocha", "test:chrome:headful": "cross-env HEADLESS=false mocha", "prepublishOnly": "npm run build", "prepare": "node typescript-if-required.js && husky install", diff --git a/test/README.md b/test/README.md index 25da7120c95..6197547356b 100644 --- a/test/README.md +++ b/test/README.md @@ -81,5 +81,5 @@ npm run test:chrome:headful - To run tests with custom browser executable: ```bash -BINARY= npm run test:chrome # Or npm run test:firefox +BINARY= npm run test:chrome:headless # Or npm run test:firefox ``` diff --git a/utils/bisect.js b/utils/bisect.js index a7c565681d8..bbec3d24319 100755 --- a/utils/bisect.js +++ b/utils/bisect.js @@ -35,7 +35,7 @@ Usage: node bisect.js --good --bad