chore: improve CI jobs (#9054)

This PR separates the installations tests from normal tests and changes
the matrices as follows:

- Installations tests on all platforms and the latest 3 node versions.
- Chrome tests are done on all platforms on the latest node version.
- Firefox tests are done only on linux on the latest node version.

.github/workflows/ci.yml

@@ -27,14 +27,10 @@ jobs:
       - name: Set up Node.js
         uses: actions/setup-node@v3.4.1
         with:
-          node-version: 18
+          cache: npm
-      - name: Setup cache for Chromium binary
+          node-version: latest
-        uses: actions/cache@v3
-        with:
-          path: .local-chromium
-          key: ${{ runner.os }}-chromium-${{ hashFiles('src/revisions.ts') }}
       - name: Install dependencies
-        run: npm ci
+        run: npm ci --ignore-scripts
       - name: Check code
         run: npm run check
       - name: Lint code
@@ -57,13 +53,9 @@ jobs:
         uses: actions/setup-node@v3.4.1
         with:
           cache: npm
-      - name: Setup cache for Chromium binary
+          node-version: latest
-        uses: actions/cache@v3
-        with:
-          path: packages/puppeteer-core/.local-chromium
-          key: ${{ runner.os }}-chromium-${{ hashFiles('src/revisions.ts') }}
       - name: Install dependencies
-        run: npm ci
+        run: npm ci --ignore-scripts
       - name: Build
         run: npm run docs
       - name: Check if autogenerated docs differ
@@ -101,6 +93,7 @@ jobs:
         uses: actions/setup-node@v3.4.1
         with:
           cache: npm
+          node-version: latest
       - name: Install dependencies
         working-directory: ./website
         run: npm ci
@@ -117,157 +110,148 @@ jobs:
           user_name: release-please[bot]
           user_email: 55107282+release-please[bot]@users.noreply.github.com
 
-  tests:
+  chrome-tests:
-    name: ${{ matrix.spec.name }} tests on ${{ matrix.spec.machine }}(${{ matrix.spec.node }})
+    name: ${{ matrix.suite }} tests on ${{ matrix.os }}
-    runs-on: ${{ matrix.spec.machine }}
+    runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        spec:
+        os:
-          - name: chrome-headless
+          - ubuntu-latest
-            machine: ubuntu-latest
+          - windows-latest
-            xvfb: true
+          - macos-latest
-            node: 14
+        suite:
-            suite: chrome-headless
+          - chrome-headless
-          - name: chrome-headless
+          - chrome-headful
-            machine: ubuntu-latest
+          - chrome-new-headless
-            xvfb: true
-            node: 16
-            suite: chrome-headless
-          - name: chrome-headless
-            machine: ubuntu-latest
-            xvfb: true
-            node: 18
-            suite: chrome-headless
-          - name: chrome-headless
-            machine: windows-latest
-            xvfb: false
-            node: 18
-            suite: chrome-headless
-          - name: chrome-headless
-            machine: macos-latest
-            xvfb: false
-            node: 18
-            suite: chrome-headless
-          - name: chrome-headful
-            machine: ubuntu-latest
-            xvfb: true
-            node: 14
-            suite: chrome-headful
-          - name: chrome-headful
-            machine: ubuntu-latest
-            xvfb: true
-            node: 16
-            suite: chrome-headful
-          - name: chrome-headful
-            machine: ubuntu-latest
-            xvfb: true
-            node: 18
-            suite: chrome-headful
-          - name: chrome-new-headless
-            machine: ubuntu-latest
-            xvfb: true
-            node: 14
-            suite: chrome-new-headless
-          - name: chrome-new-headless
-            machine: ubuntu-latest
-            xvfb: true
-            node: 16
-            suite: chrome-new-headless
-          - name: chrome-new-headless
-            machine: ubuntu-latest
-            xvfb: true
-            node: 18
-            suite: chrome-new-headless
-          - name: firefox-headless
-            machine: ubuntu-latest
-            xvfb: true
-            node: 14
-            suite: firefox-headless
-          - name: firefox-headless
-            machine: ubuntu-latest
-            xvfb: true
-            node: 16
-            suite: firefox-headless
-          - name: firefox-headless
-            machine: ubuntu-latest
-            xvfb: true
-            node: 18
-            suite: firefox-headless
-          - name: firefox-headful
-            machine: ubuntu-latest
-            xvfb: true
-            node: 16
-            suite: firefox-headful
-          - name: firefox-headful
-            machine: ubuntu-latest
-            xvfb: true
-            node: 18
-            suite: firefox-headful
-          - name: firefox-bidi
-            machine: ubuntu-latest
-            xvfb: true
-            node: 18
-            suite: firefox-bidi
-          - name: firefox-headless
-            machine: macos-latest
-            xvfb: false
-            node: 18
-            suite: firefox-headless
-
     steps:
       - name: Check out repository
         uses: actions/checkout@v3.0.2
+      - name: Set up Node.js
+        uses: actions/setup-node@v3.4.1
+        with:
+          cache: npm
+          node-version: latest
+      - name: Install dependencies
+        run: npm ci --ignore-scripts
+      - name: Build libraries
+        run: npm run build
+      - name: Build tests
+        run: npm run build:test
       - name: Setup cache for Chromium binary
         uses: actions/cache@v3
         with:
-          path: .local-chromium
+          path: packages/puppeteer-core/.local-chromium
-          key: ${{ runner.os }}-chromium-${{ hashFiles('src/revisions.ts') }}
+          key: ${{ runner.os }}-chromium-${{ hashFiles('packages/puppeteer-core/src/revisions.ts') }}
+      - name: Install Chromium
+        run: npm run postinstall
+      - name: Install linux dependencies.
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        run: sudo apt-get install xvfb
+      - name: Tests types
+        run: npm run test:types
+      - name: Run all tests (for non-Linux)
+        if: ${{ matrix.os != 'ubuntu-latest' }}
+        run: npm run test -- --test-suite ${{ matrix.suite }}
+      - name: Run all tests (for Linux)
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        run: xvfb-run --auto-servernum npm run test -- --test-suite ${{ matrix.suite }}
+
+  firefox-tests:
+    name: ${{ matrix.suite }} tests on ubuntu-latest
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        suite:
+          - firefox-bidi
+          - firefox-headful
+          - firefox-headless
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Set up Node.js
+        uses: actions/setup-node@v3.4.1
+        with:
+          cache: npm
+          node-version: latest
+      - name: Install dependencies
+        run: npm ci --ignore-scripts
+      - name: Build libraries
+        run: npm run build
+      - name: Build tests
+        run: npm run build:test
       - name: Setup cache for Firefox binary
         uses: actions/cache@v3
         with:
           path: packages/puppeteer-core/.local-firefox
-          key: ${{ runner.os }}-firefox-${{ hashFiles('src/revisions.ts') }}
+          key: ${{ runner.os }}-firefox-${{ hashFiles('packages/puppeteer-core/src/revisions.ts') }}
-      - name: Set up Node.js
-        uses: actions/setup-node@v3.4.1
-        with:
-          node-version: ${{ matrix.spec.node }}
-      - name: Update NPM
-        run: npm install -g npm@latest
-      - name: Install dependencies with Chromium
-        run: npm ci
       - name: Install Firefox
         env:
           PUPPETEER_PRODUCT: firefox
-        run: npm ci
+        run: npm run postinstall
       - name: Install linux dependencies.
-        if: ${{ matrix.spec.xvfb }}
         run: sudo apt-get install xvfb
-      - name: Build
+      - name: Tests types
-        run: npm run build:test
-      - name: Test types
         run: npm run test:types
-      - name: Run all tests with xvfb
+      - name: Run all tests
-        if: ${{ matrix.spec.xvfb }}
+        run: xvfb-run --auto-servernum npm run test -- --test-suite ${{ matrix.suite }}
-        run: xvfb-run --auto-servernum npm run test -- --test-suite ${{ matrix.spec.suite }}
-      - name: Run all tests without xvfb
-        if: ${{ !matrix.spec.xvfb }}
-        run: npm run test -- --test-suite ${{ matrix.spec.suite }}
-      - name: Test bundling and installation
-        if: ${{ matrix.spec.machine == 'ubuntu-latest' }}
-        run: |
-          # Test prod build instead of dev.
-          npm run clean
-          npm run build
-          # Note: this modifies package.json to test puppeteer-core, so we test this last.
-          npm run test:install
 
-  docker-tests:
+  pack-packages:
+    name: Pack packages
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Set up Node.js
+        uses: actions/setup-node@v3.4.1
+        with:
+          cache: npm
+          node-version: latest
+      - name: Install dependencies
+        run: npm ci --ignore-scripts
+      - name: Build libraries
+        run: npm run build
+      - name: Pack libraries
+        run: npm pack --workspaces
+      - name: Upload packages
+        uses: actions/upload-artifact@v3
+        with:
+          name: packages
+          path: ./*.tgz
+
+  install-tests:
+    name: Test installation on ${{ matrix.os }} (${{ matrix.node }})
+    needs: pack-packages
     runs-on: ${{ matrix.os }}
     strategy:
+      fail-fast: false
       matrix:
-        os: [ubuntu-latest]
+        os:
-        node: [16]
+          - ubuntu-latest
+          - macos-latest
+        node:
+          - 14
+          - 16
+          - 18
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Set up Node.js
+        uses: actions/setup-node@v3.4.1
+        with:
+          cache: npm
+          node-version: ${{ matrix.node }}
+      - name: Download packages
+        uses: actions/download-artifact@v3
+        with:
+          name: packages
+      - name: Test bundling and installation
+        run: npm run test:install
+
+  docker-tests:
+    name: Test Docker image
+    runs-on: ubuntu-latest
     steps:
       - name: Check out repository
         uses: actions/checkout@v3.0.2
@@ -276,15 +260,14 @@ jobs:
       - name: Set up Node.js
         uses: actions/setup-node@v3.4.1
         with:
+          cache: npm
           node-version: ${{ matrix.node }}
       - name: Install dependencies
-        run: |
+        run: npm ci --ignore-scripts
-          npm install
+      - name: Build packages
-          ls packages/puppeteer-core/.local-chromium
+        run: npm run build
-      - name: Build
+      - name: Pack packages
-        run: |
+        run: docker/pack.sh
-          npm run build
-          docker/pack.sh
       - name: Build docker image
         working-directory: ./docker
         run: |

test/TestExpectations.json

@@ -1996,5 +1996,17 @@
     "platforms": ["darwin", "linux", "win32"],
     "parameters": ["firefox", "webDriverBiDi"],
     "expectations": ["SKIP"]
+  },
+  {
+    "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.launch should be able to launch Firefox",
+    "platforms": ["darwin", "linux", "win32"],
+    "parameters": ["chrome"],
+    "expectations": ["FAIL"]
+  },
+  {
+    "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.launch should be able to launch Chrome",
+    "platforms": ["darwin", "linux", "win32"],
+    "parameters": ["firefox"],
+    "expectations": ["FAIL"]
   }
 ]

test/test-install.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env sh
+#!/usr/bin/env bash
 set -e
 
 # All tests are headed by a echo 'Test'.
@@ -7,8 +7,15 @@ set -e
 # 2. The install script works and correctly exits without errors
 # 3. Requiring/importing Puppeteer from Node works.
 
+# MacOS doesn't support realpath
+realpath() {
+    [[ $1 = /* ]] && echo "$1" || echo "$PWD/${1#./}"
+}
+
 ROOTDIR="$(pwd)"
+if [[ -z $(realpath puppeteer-[0-9]*.tgz) ]]; then
   npm pack --workspaces
+fi
 puppeteer_tarball=$(realpath puppeteer-[0-9]*.tgz)
 puppeteer_core_tarball=$(realpath puppeteer-core-*.tgz)