ci: split jobs (#11343)

Co-authored-by: jrandolf <101637635+jrandolf@users.noreply.github.com>
This commit is contained in:
Nikolay Vitkov 2023-11-10 14:45:37 +01:00 committed by GitHub
parent cbc33bea40
commit 9d2b65912b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 648 additions and 490 deletions

59
.github/workflows/browsers.yml vendored Normal file
View File

@ -0,0 +1,59 @@
name: '@puppeteer/browsers tests'
# Declare default permissions as read only.
permissions: read-all
on:
push:
branches:
- main
pull_request:
branches:
- '**'
concurrency:
group: puppeteer-browsers-tests-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
jobs:
check-changes:
uses: ./.github/workflows/changed-packages.yml
with:
check-mergeable-state: true
browsers-tests:
name: Browsers tests on ${{ matrix.os }}
runs-on: ${{ matrix.os }}
needs: check-changes
if: ${{ contains(fromJSON(needs.check-changes.outputs.changes), 'browsers') }}
strategy:
fail-fast: false
matrix:
os:
- ubuntu-latest
- windows-latest
- macos-latest
steps:
- name: Checkout
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
- name: Set up Node.js
uses: actions/setup-node@8f152de45cc393bb48ce5d89d36b731f54556e65 # v4.0.0
with:
cache: npm
node-version: lts/*
- name: Install dependencies
run: npm ci
env:
PUPPETEER_SKIP_DOWNLOAD: true
- name: Run tests
run: npm run test -w @puppeteer/browsers
browsers-tests-required:
name: '[Required] Test the browsers packages'
needs: [check-changes, browsers-tests]
runs-on: ubuntu-latest
if: ${{ always() }}
steps:
- if: ${{ needs.browsers-tests.result != 'success' && contains(fromJSON(needs.check-changes.outputs.changes), 'browsers') }}
run: 'exit 1'
- run: 'exit 0'

View File

@ -1,480 +0,0 @@
name: CI
# Declare default permissions as read only.
permissions: read-all
on:
push:
branches:
- main
pull_request:
branches:
- '**'
concurrency:
group: ci-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
jobs:
inspect-code:
name: '[Required] Inspect code'
runs-on: ubuntu-latest
steps:
- name: Check out repository
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
fetch-depth: 2
- name: Set up Node.js
uses: actions/setup-node@8f152de45cc393bb48ce5d89d36b731f54556e65 # v4.0.0
with:
cache: npm
node-version: lts/*
- name: Install dependencies
run: npm ci
env:
PUPPETEER_SKIP_DOWNLOAD: true
# Set up GitHub Actions caching for Wireit.
- uses: google/wireit@f3a3c79c553122e2fe5829eeac7d815326502903 # setup-github-actions-caching/v1
- name: Check code
run: npm run check
- name: Validate licenses
run: npm run validate-licenses
- name: Build every package
run: npm run build
- name: Tests types
run: npm run test-types
- name: Lint code
run: npm run lint
- name: Generate documents
run: npm run docs
- name: Check if autogenerated docs differ
run: |
diff_file=$(mktemp doc_diff_XXXXXX)
git diff --color > $diff_file
if [[ -s $diff_file ]]; then
echo "Please update the documentation by running 'npm run docs'. The following was the diff"
cat $diff_file
rm $diff_file
exit 1
fi
rm $diff_file
check-changes:
uses: ./.github/workflows/changed-packages.yml
with:
check-mergeable-state: true
deploy-docs:
needs: check-changes
name: Deploy docs (if needed)
if: ${{ github.event_name != 'pull_request' && contains(fromJSON(needs.check-changes.outputs.changes), 'website') }}
runs-on: ubuntu-latest
permissions:
contents: write
steps:
- name: Check out repository
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
- name: Set up Node.js
uses: actions/setup-node@8f152de45cc393bb48ce5d89d36b731f54556e65 # v4.0.0
with:
cache: npm
node-version: lts/*
- name: Install dependencies
working-directory: ./website
run: npm ci
- name: Build website
working-directory: ./website
env:
NODE_OPTIONS: --max-old-space-size=6144
run: npm run build
- name: Deploy to GitHub Pages
uses: peaceiris/actions-gh-pages@373f7f263a76c20808c831209c920827a82a2847 # v3.9.3
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
publish_dir: ./website/build
user_name: release-please[bot]
user_email: 55107282+release-please[bot]@users.noreply.github.com
- name: Test Algolia if Crawler is blocked
env:
CRAWLER_USER_ID: ${{secrets.ALGOLIA_CRAWLER_USER_ID}}
CRAWLER_API_KEY: ${{secrets.ALGOLIA_CRAWLER_API_KEY}}
CRAWLER_ID: ${{secrets.ALGOLIA_CRAWLER_ID}}
run: |
RESPONSE=$(curl -H "Content-Type: application/json" -X GET --user "$CRAWLER_USER_ID:$CRAWLER_API_KEY" \
"https://crawler.algolia.com/api/1/crawlers/$CRAWLER_ID" | grep "\"blocked\":true" || true ); \
if [ ! -z "$RESPONSE" ]; then echo "Please go to https://crawler.algolia.com/" && exit 1; fi
- name: Trigger Algolia reindexing
env:
CRAWLER_USER_ID: ${{secrets.ALGOLIA_CRAWLER_USER_ID}}
CRAWLER_API_KEY: ${{secrets.ALGOLIA_CRAWLER_API_KEY}}
CRAWLER_ID: ${{secrets.ALGOLIA_CRAWLER_ID}}
run: |
curl -H "Content-Type: application/json" -X POST --user "$CRAWLER_USER_ID:$CRAWLER_API_KEY" \
"https://crawler.algolia.com/api/1/crawlers/$CRAWLER_ID/reindex"
doctest:
name: Doctest
runs-on: ubuntu-latest
needs: check-changes
if: ${{ false && contains(fromJSON(needs.check-changes.outputs.changes), 'puppeteer') }}
steps:
- name: Check out repository
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
- name: Set up Node.js
uses: actions/setup-node@8f152de45cc393bb48ce5d89d36b731f54556e65 # v4.0.0
with:
cache: npm
node-version: lts/*
- name: Install dependencies
run: npm ci
env:
PUPPETEER_SKIP_DOWNLOAD: true
- name: Run tests
run: npm run doctest
chrome-tests:
name: ${{ matrix.suite }} tests on ${{ matrix.os }} (${{ matrix.shard }})
runs-on: ${{ matrix.os }}
needs: check-changes
if: ${{ contains(fromJSON(needs.check-changes.outputs.changes), 'puppeteer') }}
strategy:
fail-fast: false
matrix:
os:
- ubuntu-latest
- windows-latest
- macos-latest
suite:
- chrome-headless
- chrome-headful
- chrome-new-headless
- chrome-bidi
shard:
- 1/2
- 2/2
exclude:
- os: windows-latest
suite: chrome-bidi
- os: macos-latest
suite: chrome-headful
steps:
- name: Check out repository
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
- name: Set up Node.js
uses: actions/setup-node@8f152de45cc393bb48ce5d89d36b731f54556e65 # v4.0.0
with:
cache: npm
node-version: lts/*
- name: Set up FFmpeg
uses: FedericoCarboni/setup-ffmpeg@5058c9851b649ced05c3e73a4fb5ef2995a89127 # v2.0.0
- name: Install dependencies
run: npm ci
env:
PUPPETEER_SKIP_DOWNLOAD: true
# Set up GitHub Actions caching for Wireit.
- uses: google/wireit@f3a3c79c553122e2fe5829eeac7d815326502903 # setup-github-actions-caching/v1
- name: Build packages
run: npm run build --workspace @puppeteer-test/test
- name: Setup cache for Chrome binary
if: ${{ matrix.suite != 'chrome-bidi' }}
uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2
with:
path: ~/.cache/puppeteer/chrome
key: ${{ runner.os }}-Chrome-${{ hashFiles('packages/puppeteer-core/src/revisions.ts') }}-${{ hashFiles('packages/puppeteer/src/node/install.ts') }}
- name: Install Chrome
if: ${{ matrix.suite != 'chrome-bidi' }}
run: npm run postinstall
- name: Setup cache for Chrome Canary binary
if: ${{ matrix.suite == 'chrome-bidi' }}
uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2
with:
path: ~/.cache/puppeteer/chrome-canary
key: ${{ runner.os }}-Chrome-Canary-${{ hashFiles('package.json') }}
- name: Install Chrome Canary
if: ${{ matrix.suite == 'chrome-bidi' }}
id: browser
run: node tools/download_chrome_bidi.mjs ~/.cache/puppeteer/chrome-canary
- name: Run all tests (for non-Linux)
if: ${{ matrix.os != 'ubuntu-latest' }}
run: npm run test -- --shard '${{ matrix.shard }}' --test-suite ${{ matrix.suite }} --save-stats-to /tmp/artifacts/${{ github.event_name }}_INSERTID.json
env:
PUPPETEER_EXECUTABLE_PATH: ${{ steps.browser.outputs.executablePath }}
- name: Run all tests (for Linux)
if: ${{ matrix.os == 'ubuntu-latest' }}
run: xvfb-run --auto-servernum npm run test -- --shard '${{ matrix.shard }}' --test-suite ${{ matrix.suite }} --save-stats-to /tmp/artifacts/${{ github.event_name }}_INSERTID.json
env:
PUPPETEER_EXECUTABLE_PATH: ${{ steps.browser.outputs.executablePath }}
- uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
if: always()
with:
name: test-results
path: /tmp/artifacts/*.json
chrome-tests-required:
name: '[Required] Chrome tests'
needs: [check-changes, chrome-tests]
runs-on: ubuntu-latest
if: ${{ always() }}
steps:
- if: ${{ needs.chrome-tests.result != 'success' && contains(fromJSON(needs.check-changes.outputs.changes), 'puppeteer') }}
run: 'exit 1'
- run: 'exit 0'
firefox-tests:
name: ${{ matrix.suite }} tests on ${{ matrix.os }} (${{ matrix.shard }})
runs-on: ${{ matrix.os }}
needs: check-changes
if: ${{ contains(fromJSON(needs.check-changes.outputs.changes), 'puppeteer') }}
strategy:
fail-fast: false
matrix:
os:
- ubuntu-latest
- macos-latest
suite:
- firefox-bidi
- firefox-headful
- firefox-headless
shard:
- 1/4
- 2/4
- 3/4
- 4/4
exclude:
- os: macos-latest
suite: firefox-headful
- os: macos-latest
suite: firefox-headless
# Disabled due to https://bugzilla.mozilla.org/show_bug.cgi?id=1843550
# The first time the browsers launches it crashes.
- os: macos-latest
suite: firefox-bidi
steps:
- name: Checkout
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
- name: Set up Node.js
uses: actions/setup-node@8f152de45cc393bb48ce5d89d36b731f54556e65 # v4.0.0
with:
cache: npm
node-version: lts/*
- name: Set up FFmpeg
uses: FedericoCarboni/setup-ffmpeg@5058c9851b649ced05c3e73a4fb5ef2995a89127 # v2.0.0
- name: Install dependencies
run: npm ci
env:
PUPPETEER_SKIP_DOWNLOAD: true
# Set up GitHub Actions caching for Wireit.
- uses: google/wireit@f3a3c79c553122e2fe5829eeac7d815326502903 # setup-github-actions-caching/v1
- name: Build packages
run: npm run build --workspace @puppeteer-test/test
- name: Setup cache for Firefox binary
uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2
with:
path: ~/.cache/puppeteer/firefox
key: ${{ runner.os }}-firefox-${{ hashFiles('packages/puppeteer-core/src/revisions.ts') }}-${{ hashFiles('packages/puppeteer/src/node/install.ts') }}
- name: Install Firefox
env:
PUPPETEER_PRODUCT: firefox
run: npm run postinstall
- name: Run all tests (for non-Linux)
if: ${{ matrix.os != 'ubuntu-latest' }}
run: npm run test -- --shard '${{ matrix.shard }}' --test-suite ${{ matrix.suite }} --save-stats-to /tmp/artifacts/${{ github.event_name }}_INSERTID.json
- name: Run all tests (for Linux)
if: ${{ matrix.os == 'ubuntu-latest' }}
run: xvfb-run --auto-servernum npm run test -- --shard '${{ matrix.shard }}' --test-suite ${{ matrix.suite }} --save-stats-to /tmp/artifacts/${{ github.event_name }}_INSERTID.json
- uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
if: always()
with:
name: test-results
path: /tmp/artifacts/*.json
firefox-tests-required:
name: '[Required] Firefox tests'
needs: [check-changes, firefox-tests]
runs-on: ubuntu-latest
if: ${{ always() }}
steps:
- if: ${{ needs.firefox-tests.result != 'success' && contains(fromJSON(needs.check-changes.outputs.changes), 'puppeteer') }}
run: 'exit 1'
- run: 'exit 0'
installation-test-build:
name: Build installation test
runs-on: ubuntu-latest
needs: check-changes
if: ${{ !startsWith(github.ref_name, 'release-please') && contains(fromJSON(needs.check-changes.outputs.changes), 'puppeteer') }}
steps:
- name: Checkout
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
- name: Set up Node.js
uses: actions/setup-node@8f152de45cc393bb48ce5d89d36b731f54556e65 # v4.0.0
with:
cache: npm
node-version: lts/*
- name: Install dependencies
run: npm ci
env:
PUPPETEER_SKIP_DOWNLOAD: true
- name: Build installation test
run: npm run build --workspace @puppeteer-test/installation
- name: Pack installation test
run: npm pack --workspace @puppeteer-test/installation
- name: Upload installation test
uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
with:
name: installation-test
path: puppeteer-test-installation-latest.tgz
installation-test:
name: Test ${{ matrix.pkg_manager }} installation on ${{ matrix.os }} (${{ matrix.node }})
needs: installation-test-build
if: ${{ !startsWith(github.ref_name, 'release-please') }}
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os:
- ubuntu-latest
- macos-latest
- windows-latest
node:
- 18
pkg_manager:
- npm
steps:
- name: Download installation test
uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
with:
name: installation-test
- name: Unpack installation test
run: tar -xf puppeteer-test-installation-latest.tgz --strip-components 1 -C .
- name: Set up Node.js
uses: actions/setup-node@8f152de45cc393bb48ce5d89d36b731f54556e65 # v4.0.0
with:
node-version: ${{ matrix.node }}
- name: Install dependencies
run: ${{ matrix.pkg_manager }} install
- name: Test
env:
PKG_MANAGER: ${{ matrix.pkg_manager }}
run: ${{ matrix.pkg_manager }} test
installation-test-required:
name: '[Required] Installation tests'
needs: [check-changes, installation-test]
runs-on: ubuntu-latest
if: ${{ always() }}
steps:
- if: ${{ needs.installation-test.result != 'success' && contains(fromJSON(needs.check-changes.outputs.changes), 'puppeteer') }}
run: 'exit 1'
- run: 'exit 0'
docker-tests:
name: '[Required] Test Docker image'
runs-on: ubuntu-latest
needs: check-changes
if: ${{ contains(fromJSON(needs.check-changes.outputs.changes), 'puppeteer') }}
steps:
- name: Check out repository
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
fetch-depth: 2
- name: Set up Node.js
uses: actions/setup-node@8f152de45cc393bb48ce5d89d36b731f54556e65 # v4.0.0
with:
cache: npm
node-version: lts/*
- name: Install dependencies
run: npm ci
env:
PUPPETEER_SKIP_DOWNLOAD: true
- name: Build packages
run: npm run build --workspace puppeteer
- name: Pack packages
run: docker/pack.sh
- name: Build docker image
working-directory: ./docker
run: |
docker build -t puppeteer-test-image .
- name: Run smoke test
working-directory: ./docker
run: |
docker run -i --init --cap-add=SYS_ADMIN --rm puppeteer-test-image node -e "`cat test/smoke-test.js`"
unit-tests:
name: '[Required] Unit tests'
runs-on: ubuntu-latest
needs: check-changes
if: ${{ contains(fromJSON(needs.check-changes.outputs.changes), 'puppeteer') }}
steps:
- name: Check out repository
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
- name: Set up Node.js
uses: actions/setup-node@8f152de45cc393bb48ce5d89d36b731f54556e65 # v4.0.0
with:
cache: npm
node-version: lts/*
- name: Install dependencies
run: npm ci
env:
PUPPETEER_SKIP_DOWNLOAD: true
- name: Run unit tests
run: |
npm run unit --w puppeteer-core -w puppeteer --if-present
ng-schematics-tests:
name: '[Required] Test Angular Schematics'
runs-on: ubuntu-latest
needs: check-changes
if: ${{ contains(fromJSON(needs.check-changes.outputs.changes), 'ng-schematics') }}
steps:
- name: Checkout
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
- name: Set up Node.js
uses: actions/setup-node@8f152de45cc393bb48ce5d89d36b731f54556e65 # v4.0.0
with:
cache: npm
node-version: lts/*
- name: Install dependencies
run: npm ci
env:
PUPPETEER_SKIP_DOWNLOAD: true
- name: Run tests
run: npm run test --workspace @puppeteer/ng-schematics
browsers-tests:
name: Browsers tests on ${{ matrix.os }}
runs-on: ${{ matrix.os }}
needs: check-changes
if: ${{ contains(fromJSON(needs.check-changes.outputs.changes), 'browsers') }}
strategy:
fail-fast: false
matrix:
os:
- ubuntu-latest
- windows-latest
- macos-latest
steps:
- name: Checkout
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
- name: Set up Node.js
uses: actions/setup-node@8f152de45cc393bb48ce5d89d36b731f54556e65 # v4.0.0
with:
cache: npm
node-version: lts/*
- name: Install dependencies
run: npm ci
env:
PUPPETEER_SKIP_DOWNLOAD: true
- name: Run tests
run: npm run test --workspace @puppeteer/browsers
browsers-tests-required:
name: '[Required] Test the browsers packages'
needs: [check-changes, browsers-tests]
runs-on: ubuntu-latest
if: ${{ always() }}
steps:
- if: ${{ needs.browsers-tests.result != 'success' && contains(fromJSON(needs.check-changes.outputs.changes), 'browsers') }}
run: 'exit 1'
- run: 'exit 0'

View File

@ -64,7 +64,7 @@ jobs:
# Set up GitHub Actions caching for Wireit.
- uses: google/wireit@f3a3c79c553122e2fe5829eeac7d815326502903 # setup-github-actions-caching/v1
- name: Build packages
run: npm run build --workspace @puppeteer-test/test
run: npm run build -w @puppeteer-test/test
- name: Setup cache for Chrome binary
if: ${{ inputs.suite != 'chrome-bidi' }}
uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2
@ -121,7 +121,7 @@ jobs:
# Set up GitHub Actions caching for Wireit.
- uses: google/wireit@f3a3c79c553122e2fe5829eeac7d815326502903 # setup-github-actions-caching/v1
- name: Build packages
run: npm run build --workspace @puppeteer-test/test
run: npm run build -w @puppeteer-test/test
- name: Setup cache for Firefox binary
uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2
with:

View File

@ -30,14 +30,14 @@ jobs:
PUPPETEER_SKIP_DOWNLOAD: true
- name: Build Puppeteer
run: |
npm run build --workspace @puppeteer/browsers
npm run build --workspace puppeteer-core
npm run build --workspace puppeteer
npm run build -w @puppeteer/browsers
npm run build -w puppeteer-core
npm run build -w puppeteer
- name: Pack Puppeteer
run: |
npm pack --workspace @puppeteer/browsers
npm pack --workspace puppeteer-core
npm pack --workspace puppeteer
npm pack -w @puppeteer/browsers
npm pack -w puppeteer-core
npm pack -w puppeteer
- name: Upload Puppeteer build
uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
with:

67
.github/workflows/documentation.yml vendored Normal file
View File

@ -0,0 +1,67 @@
name: Publish Docs
# Declare default permissions as read only.
permissions: read-all
on:
push:
branches:
- main
concurrency:
group: publish-docs-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
jobs:
check-changes:
uses: ./.github/workflows/changed-packages.yml
with:
check-mergeable-state: true
deploy-docs:
needs: check-changes
name: Deploy docs (if needed)
if: ${{ contains(fromJSON(needs.check-changes.outputs.changes), 'website') }}
runs-on: ubuntu-latest
permissions:
contents: write
steps:
- name: Check out repository
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
- name: Set up Node.js
uses: actions/setup-node@8f152de45cc393bb48ce5d89d36b731f54556e65 # v4.0.0
with:
cache: npm
node-version: lts/*
- name: Install dependencies
working-directory: ./website
run: npm ci
- name: Build website
working-directory: ./website
env:
NODE_OPTIONS: --max-old-space-size=6144
run: npm run build
- name: Deploy to GitHub Pages
uses: peaceiris/actions-gh-pages@373f7f263a76c20808c831209c920827a82a2847 # v3.9.3
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
publish_dir: ./website/build
user_name: release-please[bot]
user_email: 55107282+release-please[bot]@users.noreply.github.com
- name: Test Algolia if Crawler is blocked
env:
CRAWLER_USER_ID: ${{secrets.ALGOLIA_CRAWLER_USER_ID}}
CRAWLER_API_KEY: ${{secrets.ALGOLIA_CRAWLER_API_KEY}}
CRAWLER_ID: ${{secrets.ALGOLIA_CRAWLER_ID}}
run: |
RESPONSE=$(curl -H "Content-Type: application/json" -X GET --user "$CRAWLER_USER_ID:$CRAWLER_API_KEY" \
"https://crawler.algolia.com/api/1/crawlers/$CRAWLER_ID" | grep "\"blocked\":true" || true ); \
if [ ! -z "$RESPONSE" ]; then echo "Please go to https://crawler.algolia.com/" && exit 1; fi
- name: Trigger Algolia reindexing
env:
CRAWLER_USER_ID: ${{secrets.ALGOLIA_CRAWLER_USER_ID}}
CRAWLER_API_KEY: ${{secrets.ALGOLIA_CRAWLER_API_KEY}}
CRAWLER_ID: ${{secrets.ALGOLIA_CRAWLER_ID}}
run: |
curl -H "Content-Type: application/json" -X POST --user "$CRAWLER_USER_ID:$CRAWLER_API_KEY" \
"https://crawler.algolia.com/api/1/crawlers/$CRAWLER_ID/reindex"

83
.github/workflows/inspect-code.yml vendored Normal file
View File

@ -0,0 +1,83 @@
name: Inspect code
# Declare default permissions as read only.
permissions: read-all
on:
push:
branches:
- main
pull_request:
branches:
- '**'
concurrency:
group: inspect-code-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
jobs:
inspect-code:
name: '[Required] Inspect code'
runs-on: ubuntu-latest
steps:
- name: Check out repository
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
fetch-depth: 2
- name: Set up Node.js
uses: actions/setup-node@8f152de45cc393bb48ce5d89d36b731f54556e65 # v4.0.0
with:
cache: npm
node-version: lts/*
- name: Install dependencies
run: npm ci
env:
PUPPETEER_SKIP_DOWNLOAD: true
# Set up GitHub Actions caching for Wireit.
- uses: google/wireit@f3a3c79c553122e2fe5829eeac7d815326502903 # setup-github-actions-caching/v1
- name: Check code
run: npm run check
- name: Validate licenses
run: npm run validate-licenses
- name: Build every package
run: npm run build
- name: Lint code
run: npm run lint
- name: Generate documents
run: npm run docs
- name: Check if autogenerated docs differ
run: |
diff_file=$(mktemp doc_diff_XXXXXX)
git diff --color > $diff_file
if [[ -s $diff_file ]]; then
echo "Please update the documentation by running 'npm run docs'. The following was the diff"
cat $diff_file
rm $diff_file
exit 1
fi
rm $diff_file
check-changes:
uses: ./.github/workflows/changed-packages.yml
with:
check-mergeable-state: true
doctest:
name: Doctest
runs-on: ubuntu-latest
needs: check-changes
if: ${{ false && contains(fromJSON(needs.check-changes.outputs.changes), 'puppeteer') }}
steps:
- name: Check out repository
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
- name: Set up Node.js
uses: actions/setup-node@8f152de45cc393bb48ce5d89d36b731f54556e65 # v4.0.0
with:
cache: npm
node-version: lts/*
- name: Install dependencies
run: npm ci
env:
PUPPETEER_SKIP_DOWNLOAD: true
- name: Run tests
run: npm run doctest

View File

@ -0,0 +1,92 @@
name: Installation tests
# Declare default permissions as read only.
permissions: read-all
on:
push:
branches:
- main
pull_request:
branches:
- '**'
concurrency:
group: installation-tests-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
jobs:
check-changes:
uses: ./.github/workflows/changed-packages.yml
with:
check-mergeable-state: true
installation-test-build:
name: Build installation test
runs-on: ubuntu-latest
needs: check-changes
if: ${{ contains(fromJSON(needs.check-changes.outputs.changes), 'puppeteer') }}
steps:
- name: Checkout
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
- name: Set up Node.js
uses: actions/setup-node@8f152de45cc393bb48ce5d89d36b731f54556e65 # v4.0.0
with:
cache: npm
node-version: lts/*
- name: Install dependencies
run: npm ci
env:
PUPPETEER_SKIP_DOWNLOAD: true
- name: Build installation test
run: npm run build -w @puppeteer-test/installation
- name: Pack installation test
run: npm pack -w @puppeteer-test/installation
- name: Upload installation test
uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
with:
name: installation-test
path: puppeteer-test-installation-latest.tgz
installation-test:
name: Test ${{ matrix.pkg_manager }} installation on ${{ matrix.os }} (${{ matrix.node }})
needs: installation-test-build
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os:
- ubuntu-latest
- macos-latest
- windows-latest
node:
- 18
pkg_manager:
- npm
steps:
- name: Download installation test
uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
with:
name: installation-test
- name: Unpack installation test
run: tar -xf puppeteer-test-installation-latest.tgz --strip-components 1 -C .
- name: Set up Node.js
uses: actions/setup-node@8f152de45cc393bb48ce5d89d36b731f54556e65 # v4.0.0
with:
node-version: ${{ matrix.node }}
- name: Install dependencies
run: ${{ matrix.pkg_manager }} install
- name: Test
env:
PKG_MANAGER: ${{ matrix.pkg_manager }}
run: ${{ matrix.pkg_manager }} test
installation-test-required:
name: '[Required] Installation tests'
needs: [check-changes, installation-test]
runs-on: ubuntu-latest
if: ${{ always() }}
steps:
- if: ${{ needs.installation-test.result != 'success' && contains(fromJSON(needs.check-changes.outputs.changes), 'puppeteer') }}
run: 'exit 1'
- run: 'exit 0'

View File

@ -30,7 +30,7 @@ jobs:
NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN_RELEASE}}
run: |
npm config set '//wombat-dressing-room.appspot.com/:_authToken' $NODE_AUTH_TOKEN
npm publish --workspace packages/${GITHUB_REF_NAME%-v*}
npm publish -w packages/${GITHUB_REF_NAME%-v*}
- name: Deprecate old Puppeteer versions
if: ${{ startsWith(github.ref_name, 'puppeteer-v') }}
env:

110
.github/workflows/puppeteer-chrome.yml vendored Normal file
View File

@ -0,0 +1,110 @@
name: Puppeteer Chrome
# Declare default permissions as read only.
permissions: read-all
on:
push:
branches:
- main
pull_request:
branches:
- '**'
concurrency:
group: puppeteer-chrome-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
jobs:
check-changes:
uses: ./.github/workflows/changed-packages.yml
with:
check-mergeable-state: true
chrome-tests:
name: ${{ matrix.suite }} tests on ${{ matrix.os }} (${{ matrix.shard }})
runs-on: ${{ matrix.os }}
needs: check-changes
if: ${{ contains(fromJSON(needs.check-changes.outputs.changes), 'puppeteer') }}
strategy:
fail-fast: false
matrix:
os:
- ubuntu-latest
- windows-latest
- macos-latest
suite:
- chrome-headless
- chrome-headful
- chrome-new-headless
- chrome-bidi
shard:
- 1/2
- 2/2
exclude:
- os: windows-latest
suite: chrome-bidi
- os: macos-latest
suite: chrome-headful
steps:
- name: Check out repository
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
- name: Set up Node.js
uses: actions/setup-node@8f152de45cc393bb48ce5d89d36b731f54556e65 # v4.0.0
with:
cache: npm
node-version: lts/*
- name: Set up FFmpeg
uses: FedericoCarboni/setup-ffmpeg@5058c9851b649ced05c3e73a4fb5ef2995a89127 # v2.0.0
- name: Install dependencies
run: npm ci
env:
PUPPETEER_SKIP_DOWNLOAD: true
# Set up GitHub Actions caching for Wireit.
- uses: google/wireit@f3a3c79c553122e2fe5829eeac7d815326502903 # setup-github-actions-caching/v1
- name: Build packages
run: npm run build -w @puppeteer-test/test
- name: Setup cache for Chrome binary
if: ${{ matrix.suite != 'chrome-bidi' }}
uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2
with:
path: ~/.cache/puppeteer/chrome
key: ${{ runner.os }}-Chrome-${{ hashFiles('packages/puppeteer-core/src/revisions.ts') }}-${{ hashFiles('packages/puppeteer/src/node/install.ts') }}
- name: Install Chrome
if: ${{ matrix.suite != 'chrome-bidi' }}
run: npm run postinstall
- name: Setup cache for Chrome Canary binary
if: ${{ matrix.suite == 'chrome-bidi' }}
uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2
with:
path: ~/.cache/puppeteer/chrome-canary
key: ${{ runner.os }}-Chrome-Canary-${{ hashFiles('package.json') }}
- name: Install Chrome Canary
if: ${{ matrix.suite == 'chrome-bidi' }}
id: browser
run: node tools/download_chrome_bidi.mjs ~/.cache/puppeteer/chrome-canary
- name: Run all tests (for non-Linux)
if: ${{ matrix.os != 'ubuntu-latest' }}
run: npm run test -- --shard '${{ matrix.shard }}' --test-suite ${{ matrix.suite }} --save-stats-to /tmp/artifacts/${{ github.event_name }}_INSERTID.json
env:
PUPPETEER_EXECUTABLE_PATH: ${{ steps.browser.outputs.executablePath }}
- name: Run all tests (for Linux)
if: ${{ matrix.os == 'ubuntu-latest' }}
run: xvfb-run --auto-servernum npm run test -- --shard '${{ matrix.shard }}' --test-suite ${{ matrix.suite }} --save-stats-to /tmp/artifacts/${{ github.event_name }}_INSERTID.json
env:
PUPPETEER_EXECUTABLE_PATH: ${{ steps.browser.outputs.executablePath }}
- uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
if: always()
with:
name: test-results
path: /tmp/artifacts/*.json
chrome-tests-required:
name: '[Required] Chrome tests'
needs: [check-changes, chrome-tests]
runs-on: ubuntu-latest
if: ${{ always() }}
steps:
- if: ${{ needs.chrome-tests.result != 'success' && contains(fromJSON(needs.check-changes.outputs.changes), 'puppeteer') }}
run: 'exit 1'
- run: 'exit 0'

54
.github/workflows/puppeteer-docker.yml vendored Normal file
View File

@ -0,0 +1,54 @@
name: Puppeteer Docker
# Declare default permissions as read only.
permissions: read-all
on:
push:
branches:
- main
pull_request:
branches:
- '**'
concurrency:
group: puppeteer-docker-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
jobs:
check-changes:
uses: ./.github/workflows/changed-packages.yml
with:
check-mergeable-state: true
docker-tests:
name: '[Required] Docker image'
runs-on: ubuntu-latest
needs: check-changes
if: ${{ contains(fromJSON(needs.check-changes.outputs.changes), 'puppeteer') }}
steps:
- name: Check out repository
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
fetch-depth: 2
- name: Set up Node.js
uses: actions/setup-node@8f152de45cc393bb48ce5d89d36b731f54556e65 # v4.0.0
with:
cache: npm
node-version: lts/*
- name: Install dependencies
run: npm ci
env:
PUPPETEER_SKIP_DOWNLOAD: true
- name: Build packages
run: npm run build -w puppeteer
- name: Pack packages
run: docker/pack.sh
- name: Build docker image
working-directory: ./docker
run: |
docker build -t puppeteer-test-image .
- name: Run smoke test
working-directory: ./docker
run: |
docker run -i --init --cap-add=SYS_ADMIN --rm puppeteer-test-image node -e "`cat test/smoke-test.js`"

100
.github/workflows/puppeteer-firefox.yml vendored Normal file
View File

@ -0,0 +1,100 @@
name: Puppeteer Firefox
# Declare default permissions as read only.
permissions: read-all
on:
push:
branches:
- main
pull_request:
branches:
- '**'
concurrency:
group: puppeteer-firefox-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
jobs:
check-changes:
uses: ./.github/workflows/changed-packages.yml
with:
check-mergeable-state: true
firefox-tests:
name: ${{ matrix.suite }} tests on ${{ matrix.os }} (${{ matrix.shard }})
runs-on: ${{ matrix.os }}
needs: check-changes
if: ${{ contains(fromJSON(needs.check-changes.outputs.changes), 'puppeteer') }}
strategy:
fail-fast: false
matrix:
os:
- ubuntu-latest
- macos-latest
suite:
- firefox-bidi
- firefox-headful
- firefox-headless
shard:
- 1/4
- 2/4
- 3/4
- 4/4
exclude:
- os: macos-latest
suite: firefox-headful
- os: macos-latest
suite: firefox-headless
# Disabled due to https://bugzilla.mozilla.org/show_bug.cgi?id=1843550
# The first time the browsers launches it crashes.
- os: macos-latest
suite: firefox-bidi
steps:
- name: Checkout
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
- name: Set up Node.js
uses: actions/setup-node@8f152de45cc393bb48ce5d89d36b731f54556e65 # v4.0.0
with:
cache: npm
node-version: lts/*
- name: Set up FFmpeg
uses: FedericoCarboni/setup-ffmpeg@5058c9851b649ced05c3e73a4fb5ef2995a89127 # v2.0.0
- name: Install dependencies
run: npm ci
env:
PUPPETEER_SKIP_DOWNLOAD: true
# Set up GitHub Actions caching for Wireit.
- uses: google/wireit@f3a3c79c553122e2fe5829eeac7d815326502903 # setup-github-actions-caching/v1
- name: Build packages
run: npm run build -w @puppeteer-test/test
- name: Setup cache for Firefox binary
uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2
with:
path: ~/.cache/puppeteer/firefox
key: ${{ runner.os }}-firefox-${{ hashFiles('packages/puppeteer-core/src/revisions.ts') }}-${{ hashFiles('packages/puppeteer/src/node/install.ts') }}
- name: Install Firefox
env:
PUPPETEER_PRODUCT: firefox
run: npm run postinstall
- name: Run all tests (for non-Linux)
if: ${{ matrix.os != 'ubuntu-latest' }}
run: npm run test -- --shard '${{ matrix.shard }}' --test-suite ${{ matrix.suite }} --save-stats-to /tmp/artifacts/${{ github.event_name }}_INSERTID.json
- name: Run all tests (for Linux)
if: ${{ matrix.os == 'ubuntu-latest' }}
run: xvfb-run --auto-servernum npm run test -- --shard '${{ matrix.shard }}' --test-suite ${{ matrix.suite }} --save-stats-to /tmp/artifacts/${{ github.event_name }}_INSERTID.json
- uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
if: always()
with:
name: test-results
path: /tmp/artifacts/*.json
firefox-tests-required:
name: '[Required] Firefox tests'
needs: [check-changes, firefox-tests]
runs-on: ubuntu-latest
if: ${{ always() }}
steps:
- if: ${{ needs.firefox-tests.result != 'success' && contains(fromJSON(needs.check-changes.outputs.changes), 'puppeteer') }}
run: 'exit 1'
- run: 'exit 0'

67
.github/workflows/unit.yml vendored Normal file
View File

@ -0,0 +1,67 @@
name: Unit tests
# Declare default permissions as read only.
permissions: read-all
on:
push:
branches:
- main
pull_request:
branches:
- '**'
concurrency:
group: unit-tests-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
jobs:
check-changes:
uses: ./.github/workflows/changed-packages.yml
with:
check-mergeable-state: true
puppeteer:
name: '[Required] Puppeteer Unit tests'
runs-on: ubuntu-latest
needs: check-changes
if: ${{ contains(fromJSON(needs.check-changes.outputs.changes), 'puppeteer') }}
steps:
- name: Check out repository
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
- name: Set up Node.js
uses: actions/setup-node@8f152de45cc393bb48ce5d89d36b731f54556e65 # v4.0.0
with:
cache: npm
node-version: lts/*
# Set up GitHub Actions caching for Wireit.
- uses: google/wireit@f3a3c79c553122e2fe5829eeac7d815326502903 # setup-github-actions-caching/v1
- name: Install dependencies
run: npm ci
env:
PUPPETEER_SKIP_DOWNLOAD: true
- name: Run unit tests
run: |
npm run unit -w puppeteer-core -w puppeteer --if-present
- name: Tests types
run: npm run test-types
ng-schematics:
name: '[Required] Angular Schematics tests'
runs-on: ubuntu-latest
needs: check-changes
if: ${{ contains(fromJSON(needs.check-changes.outputs.changes), 'ng-schematics') }}
steps:
- name: Checkout
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
- name: Set up Node.js
uses: actions/setup-node@8f152de45cc393bb48ce5d89d36b731f54556e65 # v4.0.0
with:
cache: npm
node-version: lts/*
- name: Install dependencies
run: npm ci
env:
PUPPETEER_SKIP_DOWNLOAD: true
- name: Run tests
run: npm run test -w @puppeteer/ng-schematics

View File

@ -24,7 +24,7 @@
"prepare": "npm run prepare --workspaces --if-present",
"test": "wireit",
"test-install": "npm run test --workspace @puppeteer-test/installation",
"test-types": "tsd -t packages/puppeteer",
"test-types": "wireit",
"test:chrome": "wireit",
"test:chrome:bidi": "wireit",
"test:chrome:bidi-local": "wireit",
@ -111,6 +111,12 @@
"./test:build",
"./tools/mocha-runner:build"
]
},
"test-types": {
"command": "tsd -t packages/puppeteer",
"dependencies": [
"./packages/puppeteer:build"
]
}
},
"devDependencies": {