From d8830cbc55a07fb0cd7d3ac98d9af4e6d95b179b Mon Sep 17 00:00:00 2001 From: Alex Rudenko Date: Thu, 8 Sep 2022 12:32:39 +0200 Subject: [PATCH] chore: implement a test runner on top of mocha (#8866) * chore: implement a test runner on top of mocha This PR implements a test runner on top of mocha that performs multiple mocha runs as defined in TestSuites.json and compares the outcome of the runs against TestExpectations.json. This allows us to remove most of helpers from mocha-utils and be more flexible when defining the test configurations. --- .github/workflows/ci.yml | 22 +- .mocharc.cjs | 2 +- package-lock.json | 18 +- package.json | 13 +- test/TestExpectations.json | 1946 +++++++++++++++++ test/TestSuites.json | 41 + test/src/CDPSession.spec.ts | 4 +- test/src/EventEmitter.spec.ts | 1 + test/src/NetworkManager.spec.ts | 5 +- test/src/TargetManager.spec.ts | 5 +- test/src/accessibility.spec.ts | 6 +- test/src/ariaqueryhandler.spec.ts | 4 +- test/src/browser.spec.ts | 1 + test/src/browsercontext.spec.ts | 17 +- test/src/chromiumonly.spec.ts | 6 +- test/src/click.spec.ts | 35 +- test/src/cookies.spec.ts | 158 +- test/src/coverage.spec.ts | 6 +- test/src/defaultbrowsercontext.spec.ts | 6 +- test/src/dialog.spec.ts | 4 +- test/src/drag-and-drop.spec.ts | 4 +- test/src/elementhandle.spec.ts | 11 +- test/src/emulation.spec.ts | 19 +- test/src/evaluation.spec.ts | 113 +- test/src/fixtures.spec.ts | 5 +- test/src/frame.spec.ts | 116 +- test/src/headful.spec.ts | 74 +- test/src/idle_override.spec.ts | 4 +- test/src/ignorehttpserrors.spec.ts | 13 +- test/src/input.spec.ts | 12 +- test/src/jshandle.spec.ts | 6 +- test/src/keyboard.spec.ts | 45 +- test/src/launcher.spec.ts | 375 ++-- test/src/mocha-utils.ts | 153 +- test/src/mouse.spec.ts | 37 +- test/src/navigation.spec.ts | 333 ++- test/src/network.spec.ts | 41 +- test/src/oopif.spec.ts | 12 +- test/src/page.spec.ts | 179 +- test/src/proxy.spec.ts | 42 +- test/src/queryhandler.spec.ts | 1 + test/src/queryselector.spec.ts | 1 + .../requestinterception-experimental.spec.ts | 8 +- test/src/requestinterception.spec.ts | 8 +- test/src/screenshot.spec.ts | 100 +- test/src/target.spec.ts | 212 +- test/src/touchscreen.spec.ts | 4 +- test/src/tracing.spec.ts | 5 +- test/src/waittask.spec.ts | 39 +- test/src/worker.spec.ts | 4 +- test/tsconfig.json | 5 +- utils/mochaRunner/README.md | 47 + utils/mochaRunner/src/main.ts | 204 ++ utils/mochaRunner/src/test.ts | 46 + utils/mochaRunner/src/types.ts | 42 + utils/mochaRunner/src/utils.ts | 153 ++ utils/mochaRunner/tsconfig.json | 11 + 57 files changed, 3537 insertions(+), 1247 deletions(-) create mode 100644 test/TestExpectations.json create mode 100644 test/TestSuites.json create mode 100644 utils/mochaRunner/README.md create mode 100644 utils/mochaRunner/src/main.ts create mode 100644 utils/mochaRunner/src/test.ts create mode 100644 utils/mochaRunner/src/types.ts create mode 100644 utils/mochaRunner/src/utils.ts create mode 100644 utils/mochaRunner/tsconfig.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 363a14ce9ea..51a841f8cae 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -173,26 +173,12 @@ jobs: run: npm run build:dev - name: Test types run: npm run test:types - # On Linux we run all Chrome tests without retries and Firefox tests with retries. - - name: Run all Chrome tests (only on Linux) + - name: Run all tests with xvfb if: ${{ matrix.spec.name == 'Linux' }} - run: xvfb-run --auto-servernum npm run test:chrome - - name: Run all Firefox tests (only on Linux) - if: ${{ matrix.spec.name == 'Linux' }} - uses: nick-invision/retry@v2 - with: - command: xvfb-run --auto-servernum npm run test:firefox - timeout_minutes: 20 - max_attempts: 3 - # On other platforms we only run chrome:headless tests and continue on Firefox errors. - - name: Test Chrome Headless (not on Linux) - id: test-chrome + run: xvfb-run --auto-servernum npm run test + - name: Run all tests without xvfb if: ${{ matrix.spec.name != 'Linux' }} - run: npm run test:chrome:headless - - name: Run all Firefox tests (not on Linux) - if: ${{ matrix.spec.name != 'Linux' }} - continue-on-error: true - run: npm run test:firefox + run: npm run test - name: Test bundling and installation if: ${{ matrix.spec.name == 'Linux' }} run: | diff --git a/.mocharc.cjs b/.mocharc.cjs index e7168715d89..30bf43695b9 100644 --- a/.mocharc.cjs +++ b/.mocharc.cjs @@ -22,6 +22,6 @@ module.exports = { exit: !!process.env.CI, retries: process.env.CI ? 2 : 0, parallel: !!process.env.PARALLEL, - timeout: 25 * 1000, + timeout: 25_000, reporter: process.env.CI ? 'spec' : 'dot', }; diff --git a/package-lock.json b/package-lock.json index 70e3a1267fa..d09af77e4c1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -80,7 +80,8 @@ "text-diff": "1.0.1", "tsd": "0.22.0", "tsx": "3.8.2", - "typescript": "4.7.4" + "typescript": "4.7.4", + "zod": "3.18.0" }, "engines": { "node": ">=14.1.0" @@ -7809,6 +7810,15 @@ "optionalDependencies": { "commander": "^2.20.3" } + }, + "node_modules/zod": { + "version": "3.18.0", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.18.0.tgz", + "integrity": "sha512-gwTm8RfUCe8l9rDwN5r2A17DkAa8Ez4Yl4yXqc5VqeGaXaJahzYYXbTwvhroZi0SNBqTwh/bKm2N0mpCzuw4bA==", + "dev": true, + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } } }, "dependencies": { @@ -13501,6 +13511,12 @@ "lodash.isequal": "^4.5.0", "validator": "^13.7.0" } + }, + "zod": { + "version": "3.18.0", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.18.0.tgz", + "integrity": "sha512-gwTm8RfUCe8l9rDwN5r2A17DkAa8Ez4Yl4yXqc5VqeGaXaJahzYYXbTwvhroZi0SNBqTwh/bKm2N0mpCzuw4bA==", + "dev": true } } } diff --git a/package.json b/package.json index ef1bf03e1bd..f6e9156baaf 100644 --- a/package.json +++ b/package.json @@ -27,14 +27,14 @@ "node": ">=14.1.0" }, "scripts": { - "test": "c8 --check-coverage --lines 93 run-s test:chrome:* test:firefox", + "test": "cross-env MOZ_WEBRENDER=0 PUPPETEER_DEFERRED_PROMISE_DEBUG_TIMEOUT=20000 c8 --check-coverage --lines 93 node utils/mochaRunner/lib/main.js", "test:types": "tsd", "test:install": "scripts/test-install.sh", - "test:firefox": "cross-env PUPPETEER_PRODUCT=firefox MOZ_WEBRENDER=0 PUPPETEER_DEFERRED_PROMISE_DEBUG_TIMEOUT=20000 mocha", + "test:firefox": "npm run test -- --test-suite firefox-headless", "test:chrome": "run-s test:chrome:*", - "test:chrome:headless": "cross-env HEADLESS=true PUPPETEER_DEFERRED_PROMISE_DEBUG_TIMEOUT=20000 mocha", - "test:chrome:headless-chrome": "cross-env HEADLESS=chrome PUPPETEER_DEFERRED_PROMISE_DEBUG_TIMEOUT=20000 mocha", - "test:chrome:headful": "cross-env HEADLESS=false PUPPETEER_DEFERRED_PROMISE_DEBUG_TIMEOUT=20000 mocha", + "test:chrome:headless": "npm run test -- --test-suite chrome-headless", + "test:chrome:headless-chrome": "npm run test -- --test-suite chrome-new-headless", + "test:chrome:headful": "npm run test -- --test-suite chrome-headful", "prepublishOnly": "npm run build", "prepare": "node typescript-if-required.js && husky install", "lint": "run-s lint:prettier lint:eslint", @@ -139,6 +139,7 @@ "text-diff": "1.0.1", "tsd": "0.22.0", "tsx": "3.8.2", - "typescript": "4.7.4" + "typescript": "4.7.4", + "zod": "3.18.0" } } diff --git a/test/TestExpectations.json b/test/TestExpectations.json new file mode 100644 index 00000000000..4983051534c --- /dev/null +++ b/test/TestExpectations.json @@ -0,0 +1,1946 @@ +[ + { + "testIdPattern": "[accessibility.spec]", + "platforms": ["darwin", "win32", "linux"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[ariaqueryhandler.spec]", + "platforms": ["darwin", "win32", "linux"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[browsercontext.spec] BrowserContext should fire target events", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[browsercontext.spec] BrowserContext should isolate localStorage and cookies", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[browsercontext.spec] BrowserContext should wait for a target", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[browsercontext.spec] BrowserContext should work across sessions", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[browsercontext.spec] BrowserContext window.open should use parent tab context", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[CDPSession.spec]", + "platforms": ["darwin", "win32", "linux"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[chromiumonly.spec]", + "platforms": ["darwin", "win32", "linux"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[click.spec] Page.click should click on checkbox label and toggle", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[click.spec] Page.click should click the button if window.Node is removed", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[click.spec] Page.click should click with disabled javascript", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[cookies.spec] Cookie specs Page.cookies should get cookies from multiple urls", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[cookies.spec] Cookie specs Page.deleteCookie should work", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[cookies.spec] Cookie specs Page.setCookie should default to setting secure cookie for HTTPS websites", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[cookies.spec] Cookie specs Page.setCookie should isolate cookies in browser contexts", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[cookies.spec] Cookie specs Page.setCookie should set a cookie on a different domain", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[cookies.spec] Cookie specs Page.setCookie should set a cookie with a path", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[cookies.spec] Cookie specs Page.setCookie should set cookie with reasonable defaults", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[cookies.spec] Cookie specs Page.setCookie should set cookies from a frame", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[cookies.spec] Cookie specs Page.setCookie should set multiple cookies", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[cookies.spec] Cookie specs Page.setCookie should set secure same-site cookies from a frame", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[cookies.spec] Cookie specs Page.setCookie should work", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[coverage.spec]", + "platforms": ["darwin", "win32", "linux"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[defaultbrowsercontext.spec] DefaultBrowserContext page.deleteCookie() should work", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[defaultbrowsercontext.spec] DefaultBrowserContext page.setCookie() should work", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[dialog.spec] Page.Events.Dialog should allow accepting prompts", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[drag-and-drop.spec]", + "platforms": ["darwin", "win32", "linux"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[elementhandle.spec] ElementHandle specs ElementHandle.boundingBox should handle nested frames", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[elementhandle.spec] ElementHandle specs ElementHandle.boundingBox should handle nested frames", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[elementhandle.spec] ElementHandle specs ElementHandle.boundingBox should return null for invisible elements", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[elementhandle.spec] ElementHandle specs ElementHandle.boundingBox should return null for invisible elements", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[elementhandle.spec] ElementHandle specs ElementHandle.boundingBox should work", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[elementhandle.spec] ElementHandle specs ElementHandle.boxModel should return null for invisible elements", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[elementhandle.spec] ElementHandle specs ElementHandle.boxModel should return null for invisible elements", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[elementhandle.spec] ElementHandle specs ElementHandle.click should work", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[elementhandle.spec] ElementHandle specs ElementHandle.contentFrame should work", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[emulation.spec] Emulation Page.emulate should support clicking", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[emulation.spec] Emulation Page.emulateCPUThrottling should change the CPU throttling rate successfully", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[emulation.spec] Emulation Page.emulateCPUThrottling should change the CPU throttling rate successfully", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[emulation.spec] Emulation Page.emulateMediaFeatures should work", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[emulation.spec] Emulation Page.emulateMediaType should work", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[emulation.spec] Emulation Page.emulateNetworkConditions should change navigator.connection.effectiveType", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[emulation.spec] Emulation Page.emulateNetworkConditions should change navigator.connection.effectiveType", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[emulation.spec] Emulation Page.emulateTimezone should throw for invalid timezone IDs", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[emulation.spec] Emulation Page.emulateTimezone should throw for invalid timezone IDs", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[emulation.spec] Emulation Page.emulateTimezone should work", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[emulation.spec] Emulation Page.emulateTimezone should work", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[emulation.spec] Emulation Page.emulateVisionDeficiency should work", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[emulation.spec] Emulation Page.emulateVisionDeficiency should work", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[emulation.spec] Emulation Page.viewport should support landscape emulation", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[evaluation.spec] Evaluation specs Page.evaluate should be able to throw a tricky error", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[evaluation.spec] Evaluation specs Page.evaluate should fail for circular object", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[evaluation.spec] Evaluation specs Page.evaluate should not throw an error when evaluation does a navigation", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[evaluation.spec] Evaluation specs Page.evaluate should return undefined for non-serializable objects", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[evaluation.spec] Evaluation specs Page.evaluate should return undefined for objects with symbols", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[evaluation.spec] Evaluation specs Page.evaluate should simulate a user gesture", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[evaluation.spec] Evaluation specs Page.evaluate should throw a nice error after a navigation", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[evaluation.spec] Evaluation specs Page.evaluate should throw if elementHandles are from other frames", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[evaluation.spec] Evaluation specs Page.evaluate should throw when evaluation triggers reload", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[evaluation.spec] Evaluation specs Page.evaluate should work from-inside an exposed function", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[evaluation.spec] Evaluation specs Page.evaluateOnNewDocument should evaluate before anything else on the page", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[evaluation.spec] Evaluation specs Page.evaluateOnNewDocument should evaluate before anything else on the page", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[evaluation.spec] Evaluation specs Page.evaluateOnNewDocument should work with CSP", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[evaluation.spec] Evaluation specs Page.evaluateOnNewDocument should work with CSP", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[fixtures.spec] Fixtures dumpio option should work with pipe option", + "platforms": ["linux"], + "parameters": ["chrome", "headful"], + "expectations": ["FAIL"] + }, + { + "testIdPattern": "[fixtures.spec] Fixtures dumpio option should work with pipe option", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[fixtures.spec] Fixtures dumpio option should work with pipe option", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[frame.spec] Frame specs Frame Management should handle nested frames", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[frame.spec] Frame specs Frame Management should report different frame instance when frame re-attaches", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[frame.spec] Frame specs Frame Management should report frame from-inside shadow DOM", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[frame.spec] Frame specs Frame Management should report frame.name()", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[frame.spec] Frame specs Frame Management should report frame.parent()", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[frame.spec] Frame specs Frame Management should send events when frames are manipulated dynamically", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[frame.spec] Frame specs Frame Management should support lazy frames", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[frame.spec] Frame specs Frame.evaluate should throw for detached frames", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[frame.spec] Frame specs Frame.executionContext should work", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[headful.spec]", + "platforms": ["darwin", "win32", "linux"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[idle_override.spec] Emulate idle state changing idle state emulation causes change of the IdleDetector state", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[idle_override.spec] Emulate idle state changing idle state emulation causes change of the IdleDetector state", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[idle_overrides.spec]", + "platforms": ["darwin", "win32", "linux"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[ignorehttperrors.spec]", + "platforms": ["darwin", "win32", "linux"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[ignorehttpserrors.spec] ignoreHTTPSErrors Response.securityDetails should work", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[ignorehttpserrors.spec] ignoreHTTPSErrors Response.securityDetails should work", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[ignorehttpserrors.spec] ignoreHTTPSErrors should work with mixed content", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[ignorehttpserrors.spec] ignoreHTTPSErrors should work with request interception", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[ignorehttpserrors.spec] ignoreHTTPSErrors Response.securityDetails Network redirects should report SecurityDetails", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["PASS", "FAIL"] + }, + { + "testIdPattern": "[input.spec]", + "platforms": ["darwin", "win32", "linux"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[jshandle.spec] JSHandle JSHandle.click should work", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[jshandle.spec] JSHandle JSHandle.jsonValue should not work with dates", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[jshandle.spec] JSHandle JSHandle.jsonValue should not work with dates", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[jshandle.spec] JSHandle JSHandleonValue should not work with dates", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[keyboard.spec] Keyboard ElementHandle.press should support |text| option", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[keyboard.spec] Keyboard should press the meta key", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[keyboard.spec] Keyboard should press the metaKey", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[keyboard.spec] Keyboard should report shiftKey", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[keyboard.spec] Keyboard should send a character with sendCharacter", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[keyboard.spec] Keyboard should specify location", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[keyboard.spec] Keyboard should specify repeat property", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[keyboard.spec] Keyboard should type all kinds of characters", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[keyboard.spec] Keyboard should type emoji", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[keyboard.spec] Keyboard should type emoji into an iframe", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[launcher.spec] Launcher specs Browser target events should work", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[launcher.spec] Launcher specs Browser.Events.disconnected should be emitted when: browser gets closed, disconnected or underlying websocket gets closed", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.connect should be able to connect to the same page simultaneously", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.connect should be able to reconnect to a disconnected browser", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.connect should support targetFilter option", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.executablePath when the product is chrome, platform is not darwin, and arch is arm64 and the executable does not exist does not return /usr/bin/chromium-browser", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.executablePath when the product is chrome, platform is not darwin, and arch is arm64 and the executable exists returns /usr/bin/chromium-browser", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.launch should filter out ignored default arguments in Chrome", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.launch should have custom URL when launching browser", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.launch should launch Chrome properly with --no-startup-window and waitForInitialPage=false", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.launch tmp profile should be cleaned up", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.launch userDataDir argument with non-existent dir", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[mouse.spec] Mouse should send mouse wheel events", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[mouse.spec] Mouse should trigger hover state", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[mouse.spec] Mouse should trigger hover state with removed window.Node", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[mouse.spec] Mouse should tween mouse movement", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[navigation.spec] navigation Frame.goto should navigate subframes", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[navigation.spec] navigation Frame.goto should navigate subframes", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[navigation.spec] navigation Frame.goto should reject when frame detaches", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[navigation.spec] navigation Frame.goto should reject when frame detaches", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[navigation.spec] navigation Frame.goto should return matching responses", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[navigation.spec] navigation Frame.goto should return matching responses", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[navigation.spec] navigation Frame.goto should return matching responses", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[navigation.spec] navigation Frame.goto should return matching responses", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[navigation.spec] navigation Frame.goto should return matching responses", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[navigation.spec] navigation Frame.goto should return matching responses", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[navigation.spec] navigation Frame.waitForNavigation should fail when frame detaches", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[navigation.spec] navigation Frame.waitForNavigation should fail when frame detaches", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[navigation.spec] navigation Frame.waitForNavigation should work", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[navigation.spec] navigation Frame.waitForNavigation should work", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[navigation.spec] navigation Page.goBack should work with HistoryAPI", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[navigation.spec] navigation Page.goto should fail when navigating to bad SSL", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[navigation.spec] navigation Page.goto should fail when navigating to bad url", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[navigation.spec] navigation Page.goto should fail when server returns 204", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[navigation.spec] navigation Page.goto should navigate to dataURL and fire dataURL requests", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[navigation.spec] navigation Page.goto should navigate to empty page with networkidle0", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[navigation.spec] navigation Page.goto should navigate to empty page with networkidle2", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[navigation.spec] navigation Page.goto should navigate to URL with hash and fire requests without hash", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[navigation.spec] navigation Page.goto should send referer", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[navigation.spec] navigation Page.goto should wait for network idle to succeed navigation", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[navigation.spec] navigation Page.goto should work when navigating to data url", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[navigation.spec] navigation Page.goto should work with subframes return 204", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[navigation.spec] navigation Page.waitForNavigation should work when subframe issues window.stop()", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[navigation.spec] navigation Page.waitForNavigation should work with DOM history.back()/history.forward()", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[navigation.spec] navigation Page.waitForNavigation should work with history.pushState()", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[navigation.spec] navigation Page.waitForNavigation should work with history.replaceState()", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network \"after each\" hook for \"should wait until response completes\"", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Network Events Page.Events.Request", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["PASS", "FAIL"] + }, + { + "testIdPattern": "[network.spec] network Network Events Page.Events.Response", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["PASS", "FAIL"] + }, + { + "testIdPattern": "[network.spec] network Network Events Page.Events.RequestFailed", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Network Events Page.Events.RequestFinished", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Network Events Page.Events.RequestServedFromCache", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Network Events should fire events in proper order", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Network Events should support redirects", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Page.authenticate should allow disable authentication", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Page.authenticate should fail if wrong credentials", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Page.authenticate should not disable caching", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Page.authenticate should work", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Page.setExtraHTTPHeaders should work", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network raw network headers Cross-origin set-cookie", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["FAIL", "PASS"] + }, + { + "testIdPattern": "[network.spec] network raw network headers Same-origin set-cookie subresource", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Request.frame should work for subframe navigation request", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Request.headers should define Chrome as user agent header", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Request.headers should define Firefox as user agent header", + "platforms": ["darwin", "win32", "linux"], + "parameters": ["chrome"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Request.initiator should return the initiator", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Request.initiator should return the initiator", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Request.isNavigationRequest should work", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Request.isNavigationRequest should work when navigating to image", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Request.isNavigationRequest should work with request interception", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Request.postData should work", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Request.postData should work", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Response.buffer should throw if the response does not have a body", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Response.buffer should work", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Response.buffer should work with compression", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Response.fromCache should work", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Response.fromCache should work", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Response.fromServiceWorker Response.fromServiceWorker", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Response.fromServiceWorker Response.fromServiceWorker", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Response.json should work", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Response.text should return uncompressed text", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Response.text should return uncompressed text", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Response.text should throw when requesting body of redirected response", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Response.text should throw when requesting body of redirected response", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Response.text should wait until response completes", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Response.text should wait until response completes", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Response.text should work", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Response.text should work", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Response.timing returns timing information", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[oopif.spec]", + "platforms": ["darwin", "win32", "linux"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[oopif.spec] OOPIF \"after all\" hook for \"should keep track of a frames OOP state\"", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[oopif.spec] OOPIF \"before all\" hook for \"should keep track of a frames OOP state\"", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[oopif.spec] OOPIF should support OOP iframes becoming normal iframes again", + "platforms": ["darwin"], + "parameters": ["chrome", "headless"], + "expectations": ["PASS", "FAIL"] + }, + { + "testIdPattern": "[oopif.spec] OOPIF should keep track of a frames OOP state", + "platforms": ["darwin"], + "parameters": ["chrome", "headless"], + "expectations": ["PASS", "FAIL"] + }, + { + "testIdPattern": "[oopif.spec] OOPIF should wait for inner OOPIFs", + "platforms": ["linux"], + "parameters": ["chrome", "headless"], + "expectations": ["PASS", "FAIL"] + }, + { + "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.connect should be able to connect to a browser with no page targets", + "platforms": ["linux", "darwin", "win32"], + "parameters": ["chrome", "chrome-headless"], + "expectations": ["PASS", "FAIL"] + }, + { + "testIdPattern": "[page.spec] Page \"before each\" hook for \"should return the page title\"", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox", "headless"], + "expectations": ["PASS", "FAIL"] + }, + { + "testIdPattern": "[page.spec] Page BrowserContext.overridePermissions should deny permission when not listed", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page BrowserContext.overridePermissions should grant permission when listed", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page BrowserContext.overridePermissions should grant persistent-storage", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page BrowserContext.overridePermissions should isolate permissions between browser contexts", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page BrowserContext.overridePermissions should reset permissions", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page BrowserContext.overridePermissions should trigger permission onchange", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page ExecutionContext.queryObjects should work", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page ExecutionContext.queryObjects should work for non-blank page", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.addStyleTag should throw when added with content to the CSP page", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.close should *not* run beforeunload by default", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.close should run beforeunload if asked for", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.close should terminate network waiters", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.Events.Close should work with window.close", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.Events.Console should have location and stack trace for console API calls", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.Events.Console should have location and stack trace for console API calls", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.Events.Console should have location when fetch fails", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.Events.Console should have location when fetch fails", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.Events.Console should not fail for window object", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.Events.Console should not fail for window object", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.Events.Console should trigger correct Log", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.Events.Console should trigger correct Log", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.Events.Console should work", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.Events.Console should work", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.Events.Console should work for different console API calls", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.Events.Console should work for different console API calls", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.Events.error should throw when page crashes", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.Events.error should throw when page crashes", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.Events.Popup should work", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.Events.Popup should work", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.Events.Popup should work with clicking target=_blank and rel=noopener", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.Events.Popup should work with clicking target=_blank and rel=noopener", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.Events.Popup should work with clicking target=_blank and with rel=opener", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.Events.Popup should work with clicking target=_blank and with rel=opener", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.Events.Popup should work with clicking target=_blank and without rel=opener", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.Events.Popup should work with clicking target=_blank and without rel=opener", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.Events.Popup should work with fake-clicking target=_blank and rel=noopener", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.Events.Popup should work with fake-clicking target=_blank and rel=noopener", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.Events.Popup should work with noopener", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.Events.Popup should work with noopener", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.exposeFunction should await returned promise", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.exposeFunction should await returned promise", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.exposeFunction should be callable from-inside evaluateOnNewDocument", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.exposeFunction should be callable from-inside evaluateOnNewDocument", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.exposeFunction should fallback to default export when passed a module object", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.exposeFunction should fallback to default export when passed a module object", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.exposeFunction should not throw when frames detach", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.exposeFunction should not throw when frames detach", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.exposeFunction should support throwing \"null\"", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.exposeFunction should support throwing \"null\"", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.exposeFunction should survive navigation", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.exposeFunction should survive navigation", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.exposeFunction should throw exception in page context", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.exposeFunction should throw exception in page context", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.exposeFunction should work", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.exposeFunction should work", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.exposeFunction should work on frames", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.exposeFunction should work on frames", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.exposeFunction should work on frames before navigation", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.exposeFunction should work on frames before navigation", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.exposeFunction should work with complex objects", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.exposeFunction should work with complex objects", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.metrics metrics event fired on console.timeStamp", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.metrics metrics event fired on console.timeStamp", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.metrics should get metrics from a page", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.metrics should get metrics from a page", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.select should work when re-defining top-level Event class", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.setBypassCSP should bypass after cross-process navigation", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.setBypassCSP should bypass after cross-process navigation", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.setBypassCSP should bypass CSP header", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.setBypassCSP should bypass CSP header", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.setBypassCSP should bypass CSP in iframes as well", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.setBypassCSP should bypass CSP in iframes as well", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.setBypassCSP should bypass CSP meta tag", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.setBypassCSP should bypass CSP meta tag", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.setCacheEnabled should enable or disable the cache based on the state passed", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.setCacheEnabled should enable or disable the cache based on the state passed", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.setCacheEnabled should stay disabled when toggling request interception on/off", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.setGeolocation should work", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.setJavaScriptEnabled should work", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.setJavaScriptEnabled should work", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.setOfflineMode should emulate navigator.onLine", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.setOfflineMode should emulate navigator.onLine", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.setOfflineMode should work", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.setOfflineMode should work", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.setUserAgent should work with additional userAgentMetdata", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[proxy.spec]", + "platforms": ["darwin", "win32", "linux"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[proxy.spec] request proxy in incognito browser context should proxy requests when configured at context level", + "platforms": ["win32"], + "parameters": ["chrome", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[requestinterception-experimental.spec]", + "platforms": ["darwin", "win32", "linux"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[requestinterception.spec]", + "platforms": ["darwin", "win32", "linux"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[screenshot.spec] Screenshots ElementHandle.screenshot should capture full element when larger than viewport", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[screenshot.spec] Screenshots Page.screenshot should use scale for clip", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[screenshot.spec] Screenshots ElementHandle.screenshot should fail to screenshot a detached element", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[screenshot.spec] Screenshots ElementHandle.screenshot should work for an element with an offset", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[screenshot.spec] Screenshots ElementHandle.screenshot should work with a null viewport", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[screenshot.spec] Screenshots ElementHandle.screenshot should work with a rotated element", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[screenshot.spec] Screenshots Page.screenshot should allow transparency", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[screenshot.spec] Screenshots Page.screenshot should clip rect", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[screenshot.spec] Screenshots Page.screenshot should get screenshot bigger than the viewport", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[screenshot.spec] Screenshots Page.screenshot should render white background on jpeg file", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[screenshot.spec] Screenshots Page.screenshot should return base64", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[screenshot.spec] Screenshots Page.screenshot should take fullPage screenshots", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[screenshot.spec] Screenshots Page.screenshot should work", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[screenshot.spec] Screenshots Page.screenshot should work in \"fromSurface: false\" mode", + "platforms": ["darwin", "win32", "linux"], + "parameters": ["chrome", "headless"], + "expectations": ["FAIL"] + }, + { + "testIdPattern": "[screenshot.spec] Screenshots Page.screenshot should work in \"fromSurface: false\" mode", + "platforms": ["darwin", "win32", "linux"], + "parameters": ["chrome", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[screenshot.spec] Screenshots Page.screenshot should work with webp", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[target.spec] Target Browser.waitForTarget should wait for a target", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[target.spec] Target should be able to use async waitForTarget", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[target.spec] Target should create a worker from a service worker", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[target.spec] Target should create a worker from a shared worker", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[target.spec] Target should have an opener", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[target.spec] Target should not crash while redirecting if original request was missed", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[target.spec] Target should not report uninitialized pages", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[target.spec] Target should report when a new page is created and closed", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[target.spec] Target should report when a service worker is created and destroyed", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[target.spec] Target should report when a target url changes", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[TargetManager.spec]", + "platforms": ["darwin", "win32", "linux"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[touchscreen.spec]", + "platforms": ["darwin", "win32", "linux"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[tracing.spec]", + "platforms": ["darwin", "win32", "linux"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[waittask.spec] waittask specs Frame.waitForFunction should work with strict CSP policy", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[waittask.spec] waittask specs Frame.waitForSelector Page.waitForSelector is shortcut for main frame", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[waittask.spec] waittask specs Frame.waitForSelector should run in specified frame", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[waittask.spec] waittask specs Frame.waitForSelector should throw when frame is detached", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[waittask.spec] waittask specs Frame.waitForSelector should work with removed MutationObserver", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[waittask.spec] waittask specs Frame.waitForXPath should run in specified frame", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[waittask.spec] waittask specs Frame.waitForXPath should throw when frame is detached", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[worker.spec]", + "platforms": ["darwin", "win32", "linux"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + } +] diff --git a/test/TestSuites.json b/test/TestSuites.json new file mode 100644 index 00000000000..43d02dd09ef --- /dev/null +++ b/test/TestSuites.json @@ -0,0 +1,41 @@ +{ + "testSuites": [ + { + "id": "chrome-headless", + "platforms": ["linux", "win32", "darwin"], + "parameters": ["chrome", "headless"] + }, + { + "id": "chrome-headful", + "platforms": ["linux"], + "parameters": ["chrome", "headful"] + }, + { + "id": "chrome-new-headless", + "platforms": ["linux"], + "parameters": ["chrome", "chrome-headless"] + }, + { + "id": "firefox-headless", + "platforms": ["linux"], + "parameters": ["firefox", "headless"] + } + ], + "parameterDefinitons": { + "chrome": { + "PUPPETEER_PRODUCT": "chrome" + }, + "firefox": { + "PUPPETEER_PRODUCT": "firefox" + }, + "headless": { + "HEADLESS": "true" + }, + "headful": { + "HEADLESS": "false" + }, + "chrome-headless": { + "HEADLESS": "chrome" + } + } +} diff --git a/test/src/CDPSession.spec.ts b/test/src/CDPSession.spec.ts index 920c31051cb..a45ff2c11a6 100644 --- a/test/src/CDPSession.spec.ts +++ b/test/src/CDPSession.spec.ts @@ -20,11 +20,11 @@ import { getTestState, setupTestBrowserHooks, setupTestPageAndContextHooks, - describeChromeOnly, } from './mocha-utils.js'; import {isErrorLike} from '../../lib/cjs/puppeteer/util/ErrorLike.js'; +import {it} from './mocha-utils.js'; -describeChromeOnly('Target.createCDPSession', function () { +describe('Target.createCDPSession', function () { setupTestBrowserHooks(); setupTestPageAndContextHooks(); diff --git a/test/src/EventEmitter.spec.ts b/test/src/EventEmitter.spec.ts index 67e1ca610d8..b62e932e6e1 100644 --- a/test/src/EventEmitter.spec.ts +++ b/test/src/EventEmitter.spec.ts @@ -17,6 +17,7 @@ import {EventEmitter} from '../../lib/cjs/puppeteer/common/EventEmitter.js'; import sinon from 'sinon'; import expect from 'expect'; +import {it} from './mocha-utils.js'; describe('EventEmitter', () => { let emitter: EventEmitter; diff --git a/test/src/NetworkManager.spec.ts b/test/src/NetworkManager.spec.ts index cd8d647746f..f0d8ee795eb 100644 --- a/test/src/NetworkManager.spec.ts +++ b/test/src/NetworkManager.spec.ts @@ -14,8 +14,6 @@ * limitations under the License. */ -import {describeChromeOnly} from './mocha-utils.js'; - import expect from 'expect'; import { NetworkManager, @@ -25,12 +23,13 @@ import {HTTPRequest} from '../../lib/cjs/puppeteer/common/HTTPRequest.js'; import {EventEmitter} from '../../lib/cjs/puppeteer/common/EventEmitter.js'; import {Frame} from '../../lib/cjs/puppeteer/common/Frame.js'; import {HTTPResponse} from '../../lib/cjs/puppeteer/common/HTTPResponse.js'; +import {it} from './mocha-utils.js'; class MockCDPSession extends EventEmitter { async send(): Promise {} } -describeChromeOnly('NetworkManager', () => { +describe('NetworkManager', () => { it('should process extra info on multiple redirects', async () => { const mockCDPSession = new MockCDPSession(); new NetworkManager(mockCDPSession, true, { diff --git a/test/src/TargetManager.spec.ts b/test/src/TargetManager.spec.ts index 175cde72f32..8e39bb12d47 100644 --- a/test/src/TargetManager.spec.ts +++ b/test/src/TargetManager.spec.ts @@ -14,8 +14,9 @@ * limitations under the License. */ -import {describeChromeOnly, getTestState} from './mocha-utils'; // eslint-disable-line import/extensions +import {getTestState} from './mocha-utils'; // eslint-disable-line import/extensions import utils from './utils.js'; +import {it} from './mocha-utils.js'; import expect from 'expect'; @@ -24,7 +25,7 @@ import { BrowserContext, } from '../../lib/cjs/puppeteer/common/Browser.js'; -describeChromeOnly('TargetManager', () => { +describe('TargetManager', () => { /* We use a special browser for this test as we need the --site-per-process flag */ let browser: Browser; let context: BrowserContext; diff --git a/test/src/accessibility.spec.ts b/test/src/accessibility.spec.ts index fe066e9d01b..412f5bedba3 100644 --- a/test/src/accessibility.spec.ts +++ b/test/src/accessibility.spec.ts @@ -21,10 +21,10 @@ import { getTestState, setupTestBrowserHooks, setupTestPageAndContextHooks, - describeFailsFirefox, } from './mocha-utils.js'; +import {it} from './mocha-utils.js'; -describeFailsFirefox('Accessibility', function () { +describe('Accessibility', function () { setupTestBrowserHooks(); setupTestPageAndContextHooks(); @@ -346,7 +346,7 @@ describeFailsFirefox('Accessibility', function () { }); // Firefox does not support contenteditable="plaintext-only". - describeFailsFirefox('plaintext contenteditable', function () { + describe('plaintext contenteditable', function () { it('plain text field with role should not have children', async () => { const {page} = getTestState(); diff --git a/test/src/ariaqueryhandler.spec.ts b/test/src/ariaqueryhandler.spec.ts index 3c053490840..a34831c3f85 100644 --- a/test/src/ariaqueryhandler.spec.ts +++ b/test/src/ariaqueryhandler.spec.ts @@ -19,14 +19,14 @@ import { getTestState, setupTestBrowserHooks, setupTestPageAndContextHooks, - describeChromeOnly, } from './mocha-utils.js'; import {ElementHandle} from '../../lib/cjs/puppeteer/common/ElementHandle.js'; import utils from './utils.js'; import assert from 'assert'; +import {it} from './mocha-utils.js'; -describeChromeOnly('AriaQueryHandler', () => { +describe('AriaQueryHandler', () => { setupTestBrowserHooks(); setupTestPageAndContextHooks(); diff --git a/test/src/browser.spec.ts b/test/src/browser.spec.ts index 375b3dba934..1c06124cc5e 100644 --- a/test/src/browser.spec.ts +++ b/test/src/browser.spec.ts @@ -16,6 +16,7 @@ import expect from 'expect'; import {getTestState, setupTestBrowserHooks} from './mocha-utils.js'; +import {it} from './mocha-utils.js'; describe('Browser specs', function () { setupTestBrowserHooks(); diff --git a/test/src/browsercontext.spec.ts b/test/src/browsercontext.spec.ts index d7d5d6adaa1..ad4d3683f75 100644 --- a/test/src/browsercontext.spec.ts +++ b/test/src/browsercontext.spec.ts @@ -15,12 +15,9 @@ */ import expect from 'expect'; -import { - getTestState, - itFailsFirefox, - setupTestBrowserHooks, -} from './mocha-utils.js'; +import {getTestState, setupTestBrowserHooks} from './mocha-utils.js'; import {waitEvent} from './utils.js'; +import {it} from './mocha-utils.js'; describe('BrowserContext', function () { setupTestBrowserHooks(); @@ -60,7 +57,7 @@ describe('BrowserContext', function () { await context.close(); expect((await browser.pages()).length).toBe(1); }); - itFailsFirefox('window.open should use parent tab context', async () => { + it('window.open should use parent tab context', async () => { const {browser, server} = getTestState(); const context = await browser.createIncognitoBrowserContext(); @@ -75,7 +72,7 @@ describe('BrowserContext', function () { expect(popupTarget.browserContext()).toBe(context); await context.close(); }); - itFailsFirefox('should fire target events', async () => { + it('should fire target events', async () => { const {browser, server} = getTestState(); const context = await browser.createIncognitoBrowserContext(); @@ -99,7 +96,7 @@ describe('BrowserContext', function () { ]); await context.close(); }); - itFailsFirefox('should wait for a target', async () => { + it('should wait for a target', async () => { const {browser, puppeteer, server} = getTestState(); const context = await browser.createIncognitoBrowserContext(); @@ -156,7 +153,7 @@ describe('BrowserContext', function () { await context.close(); }); - itFailsFirefox('should isolate localStorage and cookies', async () => { + it('should isolate localStorage and cookies', async () => { const {browser, server} = getTestState(); // Create two incognito contexts. @@ -216,7 +213,7 @@ describe('BrowserContext', function () { expect(browser.browserContexts().length).toBe(1); }); - itFailsFirefox('should work across sessions', async () => { + it('should work across sessions', async () => { const {browser, puppeteer} = getTestState(); expect(browser.browserContexts().length).toBe(1); diff --git a/test/src/chromiumonly.spec.ts b/test/src/chromiumonly.spec.ts index 5cb9cd3f2e4..8bf9f00ef81 100644 --- a/test/src/chromiumonly.spec.ts +++ b/test/src/chromiumonly.spec.ts @@ -19,10 +19,10 @@ import { getTestState, setupTestBrowserHooks, setupTestPageAndContextHooks, - describeChromeOnly, } from './mocha-utils.js'; +import {it} from './mocha-utils.js'; -describeChromeOnly('Chromium-Specific Launcher tests', function () { +describe('Chromium-Specific Launcher tests', function () { describe('Puppeteer.launch |browserURL| option', function () { it('should be able to connect using browserUrl, with and without trailing slash', async () => { const {defaultBrowserOptions, puppeteer} = getTestState(); @@ -138,7 +138,7 @@ describeChromeOnly('Chromium-Specific Launcher tests', function () { }); }); -describeChromeOnly('Chromium-Specific Page Tests', function () { +describe('Chromium-Specific Page Tests', function () { setupTestBrowserHooks(); setupTestPageAndContextHooks(); it('Page.setRequestInterception should work with intervention headers', async () => { diff --git a/test/src/click.spec.ts b/test/src/click.spec.ts index bea17371286..5de40fef24c 100644 --- a/test/src/click.spec.ts +++ b/test/src/click.spec.ts @@ -19,9 +19,9 @@ import { getTestState, setupTestPageAndContextHooks, setupTestBrowserHooks, - itFailsFirefox, } from './mocha-utils.js'; import utils from './utils.js'; +import {it} from './mocha-utils.js'; describe('Page.click', function () { setupTestBrowserHooks(); @@ -52,24 +52,21 @@ describe('Page.click', function () { }) ).toBe(42); }); - itFailsFirefox( - 'should click the button if window.Node is removed', - async () => { - const {page, server} = getTestState(); + it('should click the button if window.Node is removed', async () => { + const {page, server} = getTestState(); - await page.goto(server.PREFIX + '/input/button.html'); + await page.goto(server.PREFIX + '/input/button.html'); + await page.evaluate(() => { + // @ts-expect-error Expected. + return delete window.Node; + }); + await page.click('button'); + expect( await page.evaluate(() => { - // @ts-expect-error Expected. - return delete window.Node; - }); - await page.click('button'); - expect( - await page.evaluate(() => { - return (globalThis as any).result; - }) - ).toBe('Clicked'); - } - ); + return (globalThis as any).result; + }) + ).toBe('Clicked'); + }); // @see https://github.com/puppeteer/puppeteer/issues/4281 it('should click on a span with an inline element inside', async () => { const {page} = getTestState(); @@ -110,7 +107,7 @@ describe('Page.click', function () { }) ).toBe('Clicked'); }); - itFailsFirefox('should click with disabled javascript', async () => { + it('should click with disabled javascript', async () => { const {page, server} = getTestState(); await page.setJavaScriptEnabled(false); @@ -240,7 +237,7 @@ describe('Page.click', function () { ).toBe(false); }); - itFailsFirefox('should click on checkbox label and toggle', async () => { + it('should click on checkbox label and toggle', async () => { const {page, server} = getTestState(); await page.goto(server.PREFIX + '/input/checkbox.html'); diff --git a/test/src/cookies.spec.ts b/test/src/cookies.spec.ts index cd06ba7ad2c..f493271319e 100644 --- a/test/src/cookies.spec.ts +++ b/test/src/cookies.spec.ts @@ -19,8 +19,8 @@ import { getTestState, setupTestBrowserHooks, setupTestPageAndContextHooks, - itFailsFirefox, } from './mocha-utils.js'; +import {it} from './mocha-utils.js'; describe('Cookie specs', () => { setupTestBrowserHooks(); @@ -128,7 +128,7 @@ describe('Cookie specs', () => { }, ]); }); - itFailsFirefox('should get cookies from multiple urls', async () => { + it('should get cookies from multiple urls', async () => { const {page} = getTestState(); await page.setCookie( { @@ -184,7 +184,7 @@ describe('Cookie specs', () => { }); }); describe('Page.setCookie', function () { - itFailsFirefox('should work', async () => { + it('should work', async () => { const {page, server} = getTestState(); await page.goto(server.EMPTY_PAGE); @@ -198,7 +198,7 @@ describe('Cookie specs', () => { }) ).toEqual('password=123456'); }); - itFailsFirefox('should isolate cookies in browser contexts', async () => { + it('should isolate cookies in browser contexts', async () => { const {page, server, browser} = getTestState(); const anotherContext = await browser.createIncognitoBrowserContext(); @@ -220,7 +220,7 @@ describe('Cookie specs', () => { expect(cookies2[0]!.value).toBe('page2value'); await anotherContext.close(); }); - itFailsFirefox('should set multiple cookies', async () => { + it('should set multiple cookies', async () => { const {page, server} = getTestState(); await page.goto(server.EMPTY_PAGE); @@ -257,7 +257,7 @@ describe('Cookie specs', () => { expect(cookies[0]!.session).toBe(true); expect(cookies[0]!.expires).toBe(-1); }); - itFailsFirefox('should set cookie with reasonable defaults', async () => { + it('should set cookie with reasonable defaults', async () => { const {page, server} = getTestState(); await page.goto(server.EMPTY_PAGE); @@ -288,7 +288,7 @@ describe('Cookie specs', () => { ] ); }); - itFailsFirefox('should set a cookie with a path', async () => { + it('should set a cookie with a path', async () => { const {page, server} = getTestState(); await page.goto(server.PREFIX + '/grid.html'); @@ -365,22 +365,19 @@ describe('Cookie specs', () => { 'At least one of the url and domain needs to be specified' ); }); - itFailsFirefox( - 'should default to setting secure cookie for HTTPS websites', - async () => { - const {page, server} = getTestState(); + it('should default to setting secure cookie for HTTPS websites', async () => { + const {page, server} = getTestState(); - await page.goto(server.EMPTY_PAGE); - const SECURE_URL = 'https://example.com'; - await page.setCookie({ - url: SECURE_URL, - name: 'foo', - value: 'bar', - }); - const [cookie] = await page.cookies(SECURE_URL); - expect(cookie!.secure).toBe(true); - } - ); + await page.goto(server.EMPTY_PAGE); + const SECURE_URL = 'https://example.com'; + await page.setCookie({ + url: SECURE_URL, + name: 'foo', + value: 'bar', + }); + const [cookie] = await page.cookies(SECURE_URL); + expect(cookie!.secure).toBe(true); + }); it('should be able to set unsecure cookie for HTTP website', async () => { const {page, server} = getTestState(); @@ -394,7 +391,7 @@ describe('Cookie specs', () => { const [cookie] = await page.cookies(HTTP_URL); expect(cookie!.secure).toBe(false); }); - itFailsFirefox('should set a cookie on a different domain', async () => { + it('should set a cookie on a different domain', async () => { const {page, server} = getTestState(); await page.goto(server.EMPTY_PAGE); @@ -422,7 +419,7 @@ describe('Cookie specs', () => { }, ]); }); - itFailsFirefox('should set cookies from a frame', async () => { + it('should set cookies from a frame', async () => { const {page, server} = getTestState(); await page.goto(server.PREFIX + '/grid.html'); @@ -482,71 +479,68 @@ describe('Cookie specs', () => { }, ]); }); - itFailsFirefox( - 'should set secure same-site cookies from a frame', - async () => { - const {httpsServer, puppeteer, defaultBrowserOptions} = getTestState(); + it('should set secure same-site cookies from a frame', async () => { + const {httpsServer, puppeteer, defaultBrowserOptions} = getTestState(); - const browser = await puppeteer.launch({ - ...defaultBrowserOptions, - ignoreHTTPSErrors: true, + const browser = await puppeteer.launch({ + ...defaultBrowserOptions, + ignoreHTTPSErrors: true, + }); + + const page = await browser.newPage(); + + try { + await page.goto(httpsServer.PREFIX + '/grid.html'); + await page.evaluate(src => { + let fulfill!: () => void; + const promise = new Promise(x => { + return (fulfill = x); + }); + const iframe = document.createElement('iframe'); + document.body.appendChild(iframe); + iframe.onload = fulfill; + iframe.src = src; + return promise; + }, httpsServer.CROSS_PROCESS_PREFIX); + await page.setCookie({ + name: '127-same-site-cookie', + value: 'best', + url: httpsServer.CROSS_PROCESS_PREFIX, + sameSite: 'None', }); - const page = await browser.newPage(); - - try { - await page.goto(httpsServer.PREFIX + '/grid.html'); - await page.evaluate(src => { - let fulfill!: () => void; - const promise = new Promise(x => { - return (fulfill = x); - }); - const iframe = document.createElement('iframe'); - document.body.appendChild(iframe); - iframe.onload = fulfill; - iframe.src = src; - return promise; - }, httpsServer.CROSS_PROCESS_PREFIX); - await page.setCookie({ - name: '127-same-site-cookie', - value: 'best', - url: httpsServer.CROSS_PROCESS_PREFIX, - sameSite: 'None', - }); - - expect(await page.frames()[1]!.evaluate('document.cookie')).toBe( - '127-same-site-cookie=best' - ); - expectCookieEquals( - await page.cookies(httpsServer.CROSS_PROCESS_PREFIX), - [ - { - name: '127-same-site-cookie', - value: 'best', - domain: '127.0.0.1', - path: '/', - sameParty: false, - expires: -1, - size: 24, - httpOnly: false, - sameSite: 'None', - secure: true, - session: true, - sourcePort: 443, - sourceScheme: 'Secure', - }, - ] - ); - } finally { - await page.close(); - await browser.close(); - } + expect(await page.frames()[1]!.evaluate('document.cookie')).toBe( + '127-same-site-cookie=best' + ); + expectCookieEquals( + await page.cookies(httpsServer.CROSS_PROCESS_PREFIX), + [ + { + name: '127-same-site-cookie', + value: 'best', + domain: '127.0.0.1', + path: '/', + sameParty: false, + expires: -1, + size: 24, + httpOnly: false, + sameSite: 'None', + secure: true, + session: true, + sourcePort: 443, + sourceScheme: 'Secure', + }, + ] + ); + } finally { + await page.close(); + await browser.close(); } - ); + }); }); describe('Page.deleteCookie', function () { - itFailsFirefox('should work', async () => { + it('should work', async () => { const {page, server} = getTestState(); await page.goto(server.EMPTY_PAGE); diff --git a/test/src/coverage.spec.ts b/test/src/coverage.spec.ts index 6e0fc5fe55f..930f33cd4b0 100644 --- a/test/src/coverage.spec.ts +++ b/test/src/coverage.spec.ts @@ -19,11 +19,11 @@ import { getTestState, setupTestPageAndContextHooks, setupTestBrowserHooks, - describeChromeOnly, } from './mocha-utils.js'; +import {it} from './mocha-utils.js'; describe('Coverage specs', function () { - describeChromeOnly('JSCoverage', function () { + describe('JSCoverage', function () { setupTestBrowserHooks(); setupTestPageAndContextHooks(); @@ -202,7 +202,7 @@ describe('Coverage specs', function () { }); }); - describeChromeOnly('CSSCoverage', function () { + describe('CSSCoverage', function () { setupTestBrowserHooks(); setupTestPageAndContextHooks(); diff --git a/test/src/defaultbrowsercontext.spec.ts b/test/src/defaultbrowsercontext.spec.ts index fea692932ea..f9ff849b119 100644 --- a/test/src/defaultbrowsercontext.spec.ts +++ b/test/src/defaultbrowsercontext.spec.ts @@ -19,8 +19,8 @@ import { getTestState, setupTestBrowserHooks, setupTestPageAndContextHooks, - itFailsFirefox, } from './mocha-utils.js'; +import {it} from './mocha-utils.js'; describe('DefaultBrowserContext', function () { setupTestBrowserHooks(); @@ -48,7 +48,7 @@ describe('DefaultBrowserContext', function () { }, ]); }); - itFailsFirefox('page.setCookie() should work', async () => { + it('page.setCookie() should work', async () => { const {page, server} = getTestState(); await page.goto(server.EMPTY_PAGE); @@ -78,7 +78,7 @@ describe('DefaultBrowserContext', function () { }, ]); }); - itFailsFirefox('page.deleteCookie() should work', async () => { + it('page.deleteCookie() should work', async () => { const {page, server} = getTestState(); await page.goto(server.EMPTY_PAGE); diff --git a/test/src/dialog.spec.ts b/test/src/dialog.spec.ts index 27f337eec1b..890437648ec 100644 --- a/test/src/dialog.spec.ts +++ b/test/src/dialog.spec.ts @@ -20,8 +20,8 @@ import { getTestState, setupTestPageAndContextHooks, setupTestBrowserHooks, - itFailsFirefox, } from './mocha-utils.js'; +import {it} from './mocha-utils.js'; describe('Page.Events.Dialog', function () { setupTestBrowserHooks(); @@ -46,7 +46,7 @@ describe('Page.Events.Dialog', function () { expect(dialog.message()).toBe('yo'); }); - itFailsFirefox('should allow accepting prompts', async () => { + it('should allow accepting prompts', async () => { const {page} = getTestState(); const onDialog = sinon.stub().callsFake(dialog => { diff --git a/test/src/drag-and-drop.spec.ts b/test/src/drag-and-drop.spec.ts index 0b0f93b5c91..93ae07856bf 100644 --- a/test/src/drag-and-drop.spec.ts +++ b/test/src/drag-and-drop.spec.ts @@ -19,10 +19,10 @@ import { getTestState, setupTestPageAndContextHooks, setupTestBrowserHooks, - describeChromeOnly, } from './mocha-utils.js'; +import {it} from './mocha-utils.js'; -describeChromeOnly('Input.drag', function () { +describe('Input.drag', function () { setupTestBrowserHooks(); setupTestPageAndContextHooks(); it('should throw an exception if not enabled before usage', async () => { diff --git a/test/src/elementhandle.spec.ts b/test/src/elementhandle.spec.ts index b066c8c8dbe..0d64cd48bfe 100644 --- a/test/src/elementhandle.spec.ts +++ b/test/src/elementhandle.spec.ts @@ -18,12 +18,11 @@ import expect from 'expect'; import sinon from 'sinon'; import {ElementHandle} from '../../lib/cjs/puppeteer/common/ElementHandle.js'; import { - describeFailsFirefox, getTestState, - itFailsFirefox, setupTestBrowserHooks, setupTestPageAndContextHooks, } from './mocha-utils.js'; +import {it} from './mocha-utils.js'; import utils from './utils.js'; @@ -31,7 +30,7 @@ describe('ElementHandle specs', function () { setupTestBrowserHooks(); setupTestPageAndContextHooks(); - describeFailsFirefox('ElementHandle.boundingBox', function () { + describe('ElementHandle.boundingBox', function () { it('should work', async () => { const {page, server} = getTestState(); @@ -96,7 +95,7 @@ describe('ElementHandle specs', function () { }); }); - describeFailsFirefox('ElementHandle.boxModel', function () { + describe('ElementHandle.boxModel', function () { it('should work', async () => { const {page, server} = getTestState(); @@ -162,7 +161,7 @@ describe('ElementHandle specs', function () { }); describe('ElementHandle.contentFrame', function () { - itFailsFirefox('should work', async () => { + it('should work', async () => { const {page, server} = getTestState(); await page.goto(server.EMPTY_PAGE); @@ -175,7 +174,7 @@ describe('ElementHandle specs', function () { describe('ElementHandle.click', function () { // See https://github.com/puppeteer/puppeteer/issues/7175 - itFailsFirefox('should work', async () => { + it('should work', async () => { const {page, server} = getTestState(); await page.goto(server.PREFIX + '/input/button.html'); diff --git a/test/src/emulation.spec.ts b/test/src/emulation.spec.ts index 30f6268e2e0..c275e12d24a 100644 --- a/test/src/emulation.spec.ts +++ b/test/src/emulation.spec.ts @@ -20,9 +20,8 @@ import { getTestState, setupTestBrowserHooks, setupTestPageAndContextHooks, - itFailsFirefox, - describeFailsFirefox, } from './mocha-utils.js'; +import {it} from './mocha-utils.js'; describe('Emulation', () => { setupTestBrowserHooks(); @@ -132,7 +131,7 @@ describe('Emulation', () => { }) ).toBe(true); }); - itFailsFirefox('should support landscape emulation', async () => { + it('should support landscape emulation', async () => { const {page, server} = getTestState(); await page.goto(server.PREFIX + '/mobile.html'); @@ -173,7 +172,7 @@ describe('Emulation', () => { }) ).toContain('iPhone'); }); - itFailsFirefox('should support clicking', async () => { + it('should support clicking', async () => { const {page, server} = getTestState(); await page.emulate(iPhone); @@ -192,7 +191,7 @@ describe('Emulation', () => { }); describe('Page.emulateMediaType', function () { - itFailsFirefox('should work', async () => { + it('should work', async () => { const {page} = getTestState(); expect( @@ -240,7 +239,7 @@ describe('Emulation', () => { }); describe('Page.emulateMediaFeatures', function () { - itFailsFirefox('should work', async () => { + it('should work', async () => { const {page} = getTestState(); await page.emulateMediaFeatures([ @@ -370,7 +369,7 @@ describe('Emulation', () => { }); }); - describeFailsFirefox('Page.emulateTimezone', function () { + describe('Page.emulateTimezone', function () { it('should work', async () => { const {page} = getTestState(); @@ -425,7 +424,7 @@ describe('Emulation', () => { }); }); - describeFailsFirefox('Page.emulateVisionDeficiency', function () { + describe('Page.emulateVisionDeficiency', function () { it('should work', async () => { const {page, server} = getTestState(); @@ -489,7 +488,7 @@ describe('Emulation', () => { }); }); - describeFailsFirefox('Page.emulateNetworkConditions', function () { + describe('Page.emulateNetworkConditions', function () { it('should change navigator.connection.effectiveType', async () => { const {page, puppeteer} = getTestState(); @@ -511,7 +510,7 @@ describe('Emulation', () => { }); }); - describeFailsFirefox('Page.emulateCPUThrottling', function () { + describe('Page.emulateCPUThrottling', function () { it('should change the CPU throttling rate successfully', async () => { const {page} = getTestState(); diff --git a/test/src/evaluation.spec.ts b/test/src/evaluation.spec.ts index 307bf6058fc..b68cfdc1f2c 100644 --- a/test/src/evaluation.spec.ts +++ b/test/src/evaluation.spec.ts @@ -20,9 +20,8 @@ import { getTestState, setupTestBrowserHooks, setupTestPageAndContextHooks, - itFailsFirefox, - describeFailsFirefox, } from './mocha-utils.js'; +import {it} from './mocha-utils.js'; const bigint = typeof BigInt !== 'undefined'; @@ -115,18 +114,15 @@ describe('Evaluation specs', function () { await page.goto(server.PREFIX + '/global-var.html'); expect(await page.evaluate('globalVar')).toBe(123); }); - itFailsFirefox( - 'should return undefined for objects with symbols', - async () => { - const {page} = getTestState(); + it('should return undefined for objects with symbols', async () => { + const {page} = getTestState(); - expect( - await page.evaluate(() => { - return [Symbol('foo4')]; - }) - ).toBe(undefined); - } - ); + expect( + await page.evaluate(() => { + return [Symbol('foo4')]; + }) + ).toBe(undefined); + }); it('should work with function shorthands', async () => { const {page} = getTestState(); @@ -155,7 +151,7 @@ describe('Evaluation specs', function () { ); expect(result).toBe(42); }); - itFailsFirefox('should throw when evaluation triggers reload', async () => { + it('should throw when evaluation triggers reload', async () => { const {page} = getTestState(); let error!: Error; @@ -189,7 +185,7 @@ describe('Evaluation specs', function () { await page.goto(server.EMPTY_PAGE); expect(await frameEvaluation).toBe(42); }); - itFailsFirefox('should work from-inside an exposed function', async () => { + it('should work from-inside an exposed function', async () => { const {page} = getTestState(); // Setup inpage callback, which calls Page.evaluate @@ -324,19 +320,16 @@ describe('Evaluation specs', function () { }) ).toEqual({}); }); - itFailsFirefox( - 'should return undefined for non-serializable objects', - async () => { - const {page} = getTestState(); + it('should return undefined for non-serializable objects', async () => { + const {page} = getTestState(); - expect( - await page.evaluate(() => { - return window; - }) - ).toBe(undefined); - } - ); - itFailsFirefox('should fail for circular object', async () => { + expect( + await page.evaluate(() => { + return window; + }) + ).toBe(undefined); + }); + it('should fail for circular object', async () => { const {page} = getTestState(); const result = await page.evaluate(() => { @@ -347,7 +340,7 @@ describe('Evaluation specs', function () { }); expect(result).toBe(undefined); }); - itFailsFirefox('should be able to throw a tricky error', async () => { + it('should be able to throw a tricky error', async () => { const {page} = getTestState(); const windowHandle = await page.evaluateHandle(() => { @@ -410,28 +403,25 @@ describe('Evaluation specs', function () { }); expect(error.message).toContain('JSHandle is disposed'); }); - itFailsFirefox( - 'should throw if elementHandles are from other frames', - async () => { - const {page, server} = getTestState(); + it('should throw if elementHandles are from other frames', async () => { + const {page, server} = getTestState(); - await utils.attachFrame(page, 'frame1', server.EMPTY_PAGE); - const bodyHandle = await page.frames()[1]!.$('body'); - let error!: Error; - await page - .evaluate(body => { - return body?.innerHTML; - }, bodyHandle) - .catch(error_ => { - return (error = error_); - }); - expect(error).toBeTruthy(); - expect(error.message).toContain( - 'JSHandles can be evaluated only in the context they were created' - ); - } - ); - itFailsFirefox('should simulate a user gesture', async () => { + await utils.attachFrame(page, 'frame1', server.EMPTY_PAGE); + const bodyHandle = await page.frames()[1]!.$('body'); + let error!: Error; + await page + .evaluate(body => { + return body?.innerHTML; + }, bodyHandle) + .catch(error_ => { + return (error = error_); + }); + expect(error).toBeTruthy(); + expect(error.message).toContain( + 'JSHandles can be evaluated only in the context they were created' + ); + }); + it('should simulate a user gesture', async () => { const {page} = getTestState(); const result = await page.evaluate(() => { @@ -441,7 +431,7 @@ describe('Evaluation specs', function () { }); expect(result).toBe(true); }); - itFailsFirefox('should throw a nice error after a navigation', async () => { + it('should throw a nice error after a navigation', async () => { const {page} = getTestState(); const executionContext = await page.mainFrame().executionContext(); @@ -461,19 +451,16 @@ describe('Evaluation specs', function () { }); expect((error as Error).message).toContain('navigation'); }); - itFailsFirefox( - 'should not throw an error when evaluation does a navigation', - async () => { - const {page, server} = getTestState(); + it('should not throw an error when evaluation does a navigation', async () => { + const {page, server} = getTestState(); - await page.goto(server.PREFIX + '/one-style.html'); - const result = await page.evaluate(() => { - (window as any).location = '/empty.html'; - return [42]; - }); - expect(result).toEqual([42]); - } - ); + await page.goto(server.PREFIX + '/one-style.html'); + const result = await page.evaluate(() => { + (window as any).location = '/empty.html'; + return [42]; + }); + expect(result).toEqual([42]); + }); it('should transfer 100Mb of data from page to node.js', async function () { const {page} = getTestState(); @@ -499,7 +486,7 @@ describe('Evaluation specs', function () { }); }); - describeFailsFirefox('Page.evaluateOnNewDocument', function () { + describe('Page.evaluateOnNewDocument', function () { it('should evaluate before anything else on the page', async () => { const {page, server} = getTestState(); diff --git a/test/src/fixtures.spec.ts b/test/src/fixtures.spec.ts index e20054dc6dc..4d32f732855 100644 --- a/test/src/fixtures.spec.ts +++ b/test/src/fixtures.spec.ts @@ -17,12 +17,13 @@ /* eslint-disable @typescript-eslint/no-var-requires */ import expect from 'expect'; -import {getTestState, itHeadlessOnly} from './mocha-utils.js'; +import {getTestState} from './mocha-utils.js'; import path from 'path'; +import {it} from './mocha-utils.js'; describe('Fixtures', function () { - itHeadlessOnly('dumpio option should work with pipe option', async () => { + it('dumpio option should work with pipe option', async () => { const {defaultBrowserOptions, puppeteerPath, headless} = getTestState(); if (headless === 'chrome') { // This test only works in the old headless mode. diff --git a/test/src/frame.spec.ts b/test/src/frame.spec.ts index b07b72769ab..ac010d570f5 100644 --- a/test/src/frame.spec.ts +++ b/test/src/frame.spec.ts @@ -19,18 +19,18 @@ import {CDPSession} from '../../lib/cjs/puppeteer/common/Connection.js'; import {Frame} from '../../lib/cjs/puppeteer/common/Frame.js'; import { getTestState, - itFailsFirefox, setupTestBrowserHooks, setupTestPageAndContextHooks, } from './mocha-utils.js'; import utils, {dumpFrames} from './utils.js'; +import {it} from './mocha-utils.js'; describe('Frame specs', function () { setupTestBrowserHooks(); setupTestPageAndContextHooks(); describe('Frame.executionContext', function () { - itFailsFirefox('should work', async () => { + it('should work', async () => { const {page, server} = getTestState(); await page.goto(server.EMPTY_PAGE); @@ -80,7 +80,7 @@ describe('Frame specs', function () { }); describe('Frame.evaluate', function () { - itFailsFirefox('should throw for detached frames', async () => { + it('should throw for detached frames', async () => { const {page, server} = getTestState(); const frame1 = (await utils.attachFrame( @@ -126,7 +126,7 @@ describe('Frame specs', function () { }); describe('Frame Management', function () { - itFailsFirefox('should handle nested frames', async () => { + it('should handle nested frames', async () => { const {page, server} = getTestState(); await page.goto(server.PREFIX + '/frames/nested-frames.html'); @@ -138,40 +138,37 @@ describe('Frame specs', function () { ' http://localhost:/frames/frame.html (aframe)', ]); }); - itFailsFirefox( - 'should send events when frames are manipulated dynamically', - async () => { - const {page, server} = getTestState(); + it('should send events when frames are manipulated dynamically', async () => { + const {page, server} = getTestState(); - await page.goto(server.EMPTY_PAGE); - // validate frameattached events - const attachedFrames: Frame[] = []; - page.on('frameattached', frame => { - return attachedFrames.push(frame); - }); - await utils.attachFrame(page, 'frame1', './assets/frame.html'); - expect(attachedFrames.length).toBe(1); - expect(attachedFrames[0]!.url()).toContain('/assets/frame.html'); + await page.goto(server.EMPTY_PAGE); + // validate frameattached events + const attachedFrames: Frame[] = []; + page.on('frameattached', frame => { + return attachedFrames.push(frame); + }); + await utils.attachFrame(page, 'frame1', './assets/frame.html'); + expect(attachedFrames.length).toBe(1); + expect(attachedFrames[0]!.url()).toContain('/assets/frame.html'); - // validate framenavigated events - const navigatedFrames: Frame[] = []; - page.on('framenavigated', frame => { - return navigatedFrames.push(frame); - }); - await utils.navigateFrame(page, 'frame1', './empty.html'); - expect(navigatedFrames.length).toBe(1); - expect(navigatedFrames[0]!.url()).toBe(server.EMPTY_PAGE); + // validate framenavigated events + const navigatedFrames: Frame[] = []; + page.on('framenavigated', frame => { + return navigatedFrames.push(frame); + }); + await utils.navigateFrame(page, 'frame1', './empty.html'); + expect(navigatedFrames.length).toBe(1); + expect(navigatedFrames[0]!.url()).toBe(server.EMPTY_PAGE); - // validate framedetached events - const detachedFrames: Frame[] = []; - page.on('framedetached', frame => { - return detachedFrames.push(frame); - }); - await utils.detachFrame(page, 'frame1'); - expect(detachedFrames.length).toBe(1); - expect(detachedFrames[0]!.isDetached()).toBe(true); - } - ); + // validate framedetached events + const detachedFrames: Frame[] = []; + page.on('framedetached', frame => { + return detachedFrames.push(frame); + }); + await utils.detachFrame(page, 'frame1'); + expect(detachedFrames.length).toBe(1); + expect(detachedFrames[0]!.isDetached()).toBe(true); + }); it('should send "framenavigated" when navigating on anchor URLs', async () => { const {page, server} = getTestState(); @@ -259,7 +256,7 @@ describe('Frame specs', function () { expect(detachedFrames.length).toBe(4); expect(navigatedFrames.length).toBe(1); }); - itFailsFirefox('should report frame from-inside shadow DOM', async () => { + it('should report frame from-inside shadow DOM', async () => { const {page, server} = getTestState(); await page.goto(server.PREFIX + '/shadow.html'); @@ -274,7 +271,7 @@ describe('Frame specs', function () { expect(page.frames().length).toBe(2); expect(page.frames()[1]!.url()).toBe(server.EMPTY_PAGE); }); - itFailsFirefox('should report frame.name()', async () => { + it('should report frame.name()', async () => { const {page, server} = getTestState(); await utils.attachFrame(page, 'theFrameId', server.EMPTY_PAGE); @@ -291,7 +288,7 @@ describe('Frame specs', function () { expect(page.frames()[1]!.name()).toBe('theFrameId'); expect(page.frames()[2]!.name()).toBe('theFrameName'); }); - itFailsFirefox('should report frame.parent()', async () => { + it('should report frame.parent()', async () => { const {page, server} = getTestState(); await utils.attachFrame(page, 'frame1', server.EMPTY_PAGE); @@ -300,31 +297,24 @@ describe('Frame specs', function () { expect(page.frames()[1]!.parentFrame()).toBe(page.mainFrame()); expect(page.frames()[2]!.parentFrame()).toBe(page.mainFrame()); }); - itFailsFirefox( - 'should report different frame instance when frame re-attaches', - async () => { - const {page, server} = getTestState(); + it('should report different frame instance when frame re-attaches', async () => { + const {page, server} = getTestState(); - const frame1 = await utils.attachFrame( - page, - 'frame1', - server.EMPTY_PAGE - ); - await page.evaluate(() => { - (globalThis as any).frame = document.querySelector('#frame1'); - (globalThis as any).frame.remove(); - }); - expect(frame1!.isDetached()).toBe(true); - const [frame2] = await Promise.all([ - utils.waitEvent(page, 'frameattached'), - page.evaluate(() => { - return document.body.appendChild((globalThis as any).frame); - }), - ]); - expect(frame2.isDetached()).toBe(false); - expect(frame1).not.toBe(frame2); - } - ); + const frame1 = await utils.attachFrame(page, 'frame1', server.EMPTY_PAGE); + await page.evaluate(() => { + (globalThis as any).frame = document.querySelector('#frame1'); + (globalThis as any).frame.remove(); + }); + expect(frame1!.isDetached()).toBe(true); + const [frame2] = await Promise.all([ + utils.waitEvent(page, 'frameattached'), + page.evaluate(() => { + return document.body.appendChild((globalThis as any).frame); + }), + ]); + expect(frame2.isDetached()).toBe(false); + expect(frame1).not.toBe(frame2); + }); it('should support url fragment', async () => { const {page, server} = getTestState(); @@ -335,7 +325,7 @@ describe('Frame specs', function () { server.PREFIX + '/frames/frame.html?param=value#fragment' ); }); - itFailsFirefox('should support lazy frames', async () => { + it('should support lazy frames', async () => { const {page, server} = getTestState(); await page.setViewport({width: 1000, height: 1000}); diff --git a/test/src/headful.spec.ts b/test/src/headful.spec.ts index 95abfaa2c6e..6ba729ab3f0 100644 --- a/test/src/headful.spec.ts +++ b/test/src/headful.spec.ts @@ -24,11 +24,8 @@ import { PuppeteerLaunchOptions, PuppeteerNode, } from '../../lib/cjs/puppeteer/node/Puppeteer.js'; -import { - describeChromeOnly, - getTestState, - itFailsWindows, -} from './mocha-utils.js'; +import {getTestState} from './mocha-utils.js'; +import {it} from './mocha-utils.js'; const rmAsync = promisify(rimraf); const mkdtempAsync = promisify(fs.mkdtemp); @@ -44,7 +41,7 @@ const serviceWorkerExtensionPath = path.join( 'extension' ); -describeChromeOnly('headful tests', function () { +describe('headful tests', function () { /* These tests fire up an actual browser so let's * allow a higher timeout */ @@ -214,41 +211,38 @@ describeChromeOnly('headful tests', function () { expect(pages).toEqual(['about:blank']); await browser.close(); }); - itFailsWindows( - 'headless should be able to read cookies written by headful', - async () => { - /* Needs investigation into why but this fails consistently on Windows CI. */ - const {server, puppeteer} = getTestState(); + it('headless should be able to read cookies written by headful', async () => { + /* Needs investigation into why but this fails consistently on Windows CI. */ + const {server, puppeteer} = getTestState(); - const userDataDir = await mkdtempAsync(TMP_FOLDER); - // Write a cookie in headful chrome - const headfulBrowser = await launchBrowser( - puppeteer, - Object.assign({userDataDir}, headfulOptions) - ); - const headfulPage = await headfulBrowser.newPage(); - await headfulPage.goto(server.EMPTY_PAGE); - await headfulPage.evaluate(() => { - return (document.cookie = - 'foo=true; expires=Fri, 31 Dec 9999 23:59:59 GMT'); - }); - await headfulBrowser.close(); - // Read the cookie from headless chrome - const headlessBrowser = await launchBrowser( - puppeteer, - Object.assign({userDataDir}, headlessOptions) - ); - const headlessPage = await headlessBrowser.newPage(); - await headlessPage.goto(server.EMPTY_PAGE); - const cookie = await headlessPage.evaluate(() => { - return document.cookie; - }); - await headlessBrowser.close(); - // This might throw. See https://github.com/puppeteer/puppeteer/issues/2778 - await rmAsync(userDataDir).catch(() => {}); - expect(cookie).toBe('foo=true'); - } - ); + const userDataDir = await mkdtempAsync(TMP_FOLDER); + // Write a cookie in headful chrome + const headfulBrowser = await launchBrowser( + puppeteer, + Object.assign({userDataDir}, headfulOptions) + ); + const headfulPage = await headfulBrowser.newPage(); + await headfulPage.goto(server.EMPTY_PAGE); + await headfulPage.evaluate(() => { + return (document.cookie = + 'foo=true; expires=Fri, 31 Dec 9999 23:59:59 GMT'); + }); + await headfulBrowser.close(); + // Read the cookie from headless chrome + const headlessBrowser = await launchBrowser( + puppeteer, + Object.assign({userDataDir}, headlessOptions) + ); + const headlessPage = await headlessBrowser.newPage(); + await headlessPage.goto(server.EMPTY_PAGE); + const cookie = await headlessPage.evaluate(() => { + return document.cookie; + }); + await headlessBrowser.close(); + // This might throw. See https://github.com/puppeteer/puppeteer/issues/2778 + await rmAsync(userDataDir).catch(() => {}); + expect(cookie).toBe('foo=true'); + }); // TODO: Support OOOPIF. @see https://github.com/puppeteer/puppeteer/issues/2548 xit('OOPIF: should report google.com frame', async () => { const {server, puppeteer} = getTestState(); diff --git a/test/src/idle_override.spec.ts b/test/src/idle_override.spec.ts index 89d8ecf9683..1dae41d0c91 100644 --- a/test/src/idle_override.spec.ts +++ b/test/src/idle_override.spec.ts @@ -20,10 +20,10 @@ import { getTestState, setupTestBrowserHooks, setupTestPageAndContextHooks, - describeFailsFirefox, } from './mocha-utils.js'; +import {it} from './mocha-utils.js'; -describeFailsFirefox('Emulate idle state', () => { +describe('Emulate idle state', () => { setupTestBrowserHooks(); setupTestPageAndContextHooks(); diff --git a/test/src/ignorehttpserrors.spec.ts b/test/src/ignorehttpserrors.spec.ts index 503863808c7..f4f5e6da8a3 100644 --- a/test/src/ignorehttpserrors.spec.ts +++ b/test/src/ignorehttpserrors.spec.ts @@ -22,11 +22,8 @@ import { } from '../../lib/cjs/puppeteer/common/Browser.js'; import {Page} from '../../lib/cjs/puppeteer/common/Page.js'; import {HTTPResponse} from '../../lib/cjs/puppeteer/common/HTTPResponse.js'; -import { - getTestState, - describeFailsFirefox, - itFailsFirefox, -} from './mocha-utils.js'; +import {getTestState} from './mocha-utils.js'; +import {it} from './mocha-utils.js'; describe('ignoreHTTPSErrors', function () { /* Note that this test creates its own browser rather than use @@ -59,7 +56,7 @@ describe('ignoreHTTPSErrors', function () { await context.close(); }); - describeFailsFirefox('Response.securityDetails', function () { + describe('Response.securityDetails', function () { it('should work', async () => { const {httpsServer} = getTestState(); @@ -119,7 +116,7 @@ describe('ignoreHTTPSErrors', function () { expect(error).toBeUndefined(); expect(response.ok()).toBe(true); }); - itFailsFirefox('should work with request interception', async () => { + it('should work with request interception', async () => { const {httpsServer} = getTestState(); await page.setRequestInterception(true); @@ -129,7 +126,7 @@ describe('ignoreHTTPSErrors', function () { const response = (await page.goto(httpsServer.EMPTY_PAGE))!; expect(response.status()).toBe(200); }); - itFailsFirefox('should work with mixed content', async () => { + it('should work with mixed content', async () => { const {server, httpsServer} = getTestState(); httpsServer.setRoute('/mixedcontent.html', (_req, res) => { diff --git a/test/src/input.spec.ts b/test/src/input.spec.ts index 831c362e40f..c06f4dd0b46 100644 --- a/test/src/input.spec.ts +++ b/test/src/input.spec.ts @@ -20,8 +20,8 @@ import { getTestState, setupTestBrowserHooks, setupTestPageAndContextHooks, - describeFailsFirefox, } from './mocha-utils.js'; +import {it} from './mocha-utils.js'; const FILE_TO_UPLOAD = path.join(__dirname, '/../assets/file-to-upload.txt'); @@ -29,7 +29,7 @@ describe('input tests', function () { setupTestBrowserHooks(); setupTestPageAndContextHooks(); - describeFailsFirefox('input', function () { + describe('input', function () { it('should upload the file', async () => { const {page, server} = getTestState(); @@ -76,7 +76,7 @@ describe('input tests', function () { }); }); - describeFailsFirefox('Page.waitForFileChooser', function () { + describe('Page.waitForFileChooser', function () { it('should work when file input is attached to DOM', async () => { const {page} = getTestState(); @@ -159,7 +159,7 @@ describe('input tests', function () { }); }); - describeFailsFirefox('FileChooser.accept', function () { + describe('FileChooser.accept', function () { it('should accept single file', async () => { const {page} = getTestState(); @@ -325,7 +325,7 @@ describe('input tests', function () { }); }); - describeFailsFirefox('FileChooser.cancel', function () { + describe('FileChooser.cancel', function () { it('should cancel dialog', async () => { const {page} = getTestState(); @@ -373,7 +373,7 @@ describe('input tests', function () { }); }); - describeFailsFirefox('FileChooser.isMultiple', () => { + describe('FileChooser.isMultiple', () => { it('should work for single file pick', async () => { const {page} = getTestState(); diff --git a/test/src/jshandle.spec.ts b/test/src/jshandle.spec.ts index 7147fffb99f..a97925f7781 100644 --- a/test/src/jshandle.spec.ts +++ b/test/src/jshandle.spec.ts @@ -17,11 +17,11 @@ import expect from 'expect'; import { getTestState, - itFailsFirefox, setupTestBrowserHooks, setupTestPageAndContextHooks, shortWaitForArrayToHaveAtLeastNElements, } from './mocha-utils.js'; +import {it} from './mocha-utils.js'; describe('JSHandle', function () { setupTestBrowserHooks(); @@ -159,7 +159,7 @@ describe('JSHandle', function () { expect(await bHandle.jsonValue()).toEqual(undefined); }); - itFailsFirefox('should not work with dates', async () => { + it('should not work with dates', async () => { const {page} = getTestState(); const dateHandle = await page.evaluateHandle(() => { @@ -409,7 +409,7 @@ describe('JSHandle', function () { }); describe('JSHandle.click', function () { - itFailsFirefox('should work', async () => { + it('should work', async () => { const {page} = getTestState(); const clicks: Array<[x: number, y: number]> = []; diff --git a/test/src/keyboard.spec.ts b/test/src/keyboard.spec.ts index 43ce79b207e..81270296340 100644 --- a/test/src/keyboard.spec.ts +++ b/test/src/keyboard.spec.ts @@ -21,9 +21,9 @@ import { getTestState, setupTestBrowserHooks, setupTestPageAndContextHooks, - itFailsFirefox, } from './mocha-utils.js'; import {KeyInput} from '../../lib/cjs/puppeteer/common/USKeyboardLayout.js'; +import {it} from './mocha-utils.js'; describe('Keyboard', function () { setupTestBrowserHooks(); @@ -45,7 +45,7 @@ describe('Keyboard', function () { }) ).toBe(text); }); - itFailsFirefox('should press the metaKey', async () => { + it('should press the metaKey', async () => { const {page, isFirefox} = getTestState(); await page.evaluate(() => { @@ -120,22 +120,19 @@ describe('Keyboard', function () { }) ).toBe('a'); }); - itFailsFirefox( - 'ElementHandle.press should support |text| option', - async () => { - const {page, server} = getTestState(); + it('ElementHandle.press should support |text| option', async () => { + const {page, server} = getTestState(); - await page.goto(server.PREFIX + '/input/textarea.html'); - const textarea = (await page.$('textarea'))!; - await textarea.press('a', {text: 'ё'}); - expect( - await page.evaluate(() => { - return document.querySelector('textarea')!.value; - }) - ).toBe('ё'); - } - ); - itFailsFirefox('should send a character with sendCharacter', async () => { + await page.goto(server.PREFIX + '/input/textarea.html'); + const textarea = (await page.$('textarea'))!; + await textarea.press('a', {text: 'ё'}); + expect( + await page.evaluate(() => { + return document.querySelector('textarea')!.value; + }) + ).toBe('ё'); + }); + it('should send a character with sendCharacter', async () => { const {page, server} = getTestState(); await page.goto(server.PREFIX + '/input/textarea.html'); @@ -162,7 +159,7 @@ describe('Keyboard', function () { }) ).toBe('嗨a'); }); - itFailsFirefox('should report shiftKey', async () => { + it('should report shiftKey', async () => { const {page, server} = getTestState(); await page.goto(server.PREFIX + '/input/keyboard.html'); @@ -353,7 +350,7 @@ describe('Keyboard', function () { }) ).toBe('He Wrd!'); }); - itFailsFirefox('should specify repeat property', async () => { + it('should specify repeat property', async () => { const {page, server} = getTestState(); await page.goto(server.PREFIX + '/input/textarea.html'); @@ -401,7 +398,7 @@ describe('Keyboard', function () { }) ).toBe(false); }); - itFailsFirefox('should type all kinds of characters', async () => { + it('should type all kinds of characters', async () => { const {page, server} = getTestState(); await page.goto(server.PREFIX + '/input/textarea.html'); @@ -410,7 +407,7 @@ describe('Keyboard', function () { await page.keyboard.type(text); expect(await page.evaluate('result')).toBe(text); }); - itFailsFirefox('should specify location', async () => { + it('should specify location', async () => { const {page, server} = getTestState(); await page.goto(server.PREFIX + '/input/textarea.html'); @@ -460,7 +457,7 @@ describe('Keyboard', function () { }); expect(error && error.message).toBe('Unknown key: "😊"'); }); - itFailsFirefox('should type emoji', async () => { + it('should type emoji', async () => { const {page, server} = getTestState(); await page.goto(server.PREFIX + '/input/textarea.html'); @@ -471,7 +468,7 @@ describe('Keyboard', function () { }) ).toBe('👹 Tokyo street Japan 🇯🇵'); }); - itFailsFirefox('should type emoji into an iframe', async () => { + it('should type emoji into an iframe', async () => { const {page, server} = getTestState(); await page.goto(server.EMPTY_PAGE); @@ -489,7 +486,7 @@ describe('Keyboard', function () { }) ).toBe('👹 Tokyo street Japan 🇯🇵'); }); - itFailsFirefox('should press the meta key', async () => { + it('should press the meta key', async () => { const {page, isFirefox} = getTestState(); await page.evaluate(() => { diff --git a/test/src/launcher.spec.ts b/test/src/launcher.spec.ts index a814967aa41..01e01e5cc9b 100644 --- a/test/src/launcher.spec.ts +++ b/test/src/launcher.spec.ts @@ -24,14 +24,9 @@ import {TLSSocket} from 'tls'; import {promisify} from 'util'; import {Page} from '../../lib/cjs/puppeteer/common/Page.js'; import {Product} from '../../lib/cjs/puppeteer/common/Product.js'; -import { - getTestState, - itChromeOnly, - itFailsFirefox, - itFirefoxOnly, - itOnlyRegularInstall, -} from './mocha-utils.js'; +import {getTestState, itOnlyRegularInstall} from './mocha-utils.js'; import utils from './utils.js'; +import {it} from './mocha-utils.js'; const mkdtempAsync = promisify(fs.mkdtemp); const readFileAsync = promisify(fs.readFile); @@ -251,7 +246,7 @@ describe('Launcher specs', function () { // This might throw. See https://github.com/puppeteer/puppeteer/issues/2778 await rmAsync(userDataDir).catch(() => {}); }); - itChromeOnly('tmp profile should be cleaned up', async () => { + it('tmp profile should be cleaned up', async () => { const {defaultBrowserOptions, puppeteer} = getTestState(); // Set a custom test tmp dir so that we can validate that @@ -280,7 +275,7 @@ describe('Launcher specs', function () { // Restore env var process.env['PUPPETEER_TMP_DIR'] = ''; }); - itFirefoxOnly('userDataDir option restores preferences', async () => { + it('userDataDir option restores preferences', async () => { const {defaultBrowserOptions, puppeteer} = getTestState(); const userDataDir = await mkdtempAsync(TMP_FOLDER); @@ -326,7 +321,7 @@ describe('Launcher specs', function () { // This might throw. See https://github.com/puppeteer/puppeteer/issues/2778 await rmAsync(userDataDir).catch(() => {}); }); - itChromeOnly('userDataDir argument with non-existent dir', async () => { + it('userDataDir argument with non-existent dir', async () => { const {isChrome, puppeteer, defaultBrowserOptions} = getTestState(); const userDataDir = await mkdtempAsync(TMP_FOLDER); @@ -460,49 +455,43 @@ describe('Launcher specs', function () { await page.close(); await browser.close(); }); - itChromeOnly( - 'should filter out ignored default arguments in Chrome', - async () => { - const {defaultBrowserOptions, puppeteer} = getTestState(); - // Make sure we launch with `--enable-automation` by default. - const defaultArgs = puppeteer.defaultArgs(); - const browser = await puppeteer.launch( - Object.assign({}, defaultBrowserOptions, { - // Ignore first and third default argument. - ignoreDefaultArgs: [defaultArgs[0]!, defaultArgs[2]], - }) - ); - const spawnargs = browser.process()!.spawnargs; - if (!spawnargs) { - throw new Error('spawnargs not present'); - } - expect(spawnargs.indexOf(defaultArgs[0]!)).toBe(-1); - expect(spawnargs.indexOf(defaultArgs[1]!)).not.toBe(-1); - expect(spawnargs.indexOf(defaultArgs[2]!)).toBe(-1); - await browser.close(); + it('should filter out ignored default arguments in Chrome', async () => { + const {defaultBrowserOptions, puppeteer} = getTestState(); + // Make sure we launch with `--enable-automation` by default. + const defaultArgs = puppeteer.defaultArgs(); + const browser = await puppeteer.launch( + Object.assign({}, defaultBrowserOptions, { + // Ignore first and third default argument. + ignoreDefaultArgs: [defaultArgs[0]!, defaultArgs[2]], + }) + ); + const spawnargs = browser.process()!.spawnargs; + if (!spawnargs) { + throw new Error('spawnargs not present'); } - ); - itFirefoxOnly( - 'should filter out ignored default argument in Firefox', - async () => { - const {defaultBrowserOptions, puppeteer} = getTestState(); + expect(spawnargs.indexOf(defaultArgs[0]!)).toBe(-1); + expect(spawnargs.indexOf(defaultArgs[1]!)).not.toBe(-1); + expect(spawnargs.indexOf(defaultArgs[2]!)).toBe(-1); + await browser.close(); + }); + it('should filter out ignored default argument in Firefox', async () => { + const {defaultBrowserOptions, puppeteer} = getTestState(); - const defaultArgs = puppeteer.defaultArgs(); - const browser = await puppeteer.launch( - Object.assign({}, defaultBrowserOptions, { - // Only the first argument is fixed, others are optional. - ignoreDefaultArgs: [defaultArgs[0]!], - }) - ); - const spawnargs = browser.process()!.spawnargs; - if (!spawnargs) { - throw new Error('spawnargs not present'); - } - expect(spawnargs.indexOf(defaultArgs[0]!)).toBe(-1); - expect(spawnargs.indexOf(defaultArgs[1]!)).not.toBe(-1); - await browser.close(); + const defaultArgs = puppeteer.defaultArgs(); + const browser = await puppeteer.launch( + Object.assign({}, defaultBrowserOptions, { + // Only the first argument is fixed, others are optional. + ignoreDefaultArgs: [defaultArgs[0]!], + }) + ); + const spawnargs = browser.process()!.spawnargs; + if (!spawnargs) { + throw new Error('spawnargs not present'); } - ); + expect(spawnargs.indexOf(defaultArgs[0]!)).toBe(-1); + expect(spawnargs.indexOf(defaultArgs[1]!)).not.toBe(-1); + await browser.close(); + }); it('should have default URL when launching browser', async function () { const {defaultBrowserOptions, puppeteer} = getTestState(); const browser = await puppeteer.launch(defaultBrowserOptions); @@ -512,24 +501,21 @@ describe('Launcher specs', function () { expect(pages).toEqual(['about:blank']); await browser.close(); }); - itFailsFirefox( - 'should have custom URL when launching browser', - async () => { - const {server, puppeteer, defaultBrowserOptions} = getTestState(); + it('should have custom URL when launching browser', async () => { + const {server, puppeteer, defaultBrowserOptions} = getTestState(); - const options = Object.assign({}, defaultBrowserOptions); - options.args = [server.EMPTY_PAGE].concat(options.args || []); - const browser = await puppeteer.launch(options); - const pages = await browser.pages(); - expect(pages.length).toBe(1); - const page = pages[0]!; - if (page.url() !== server.EMPTY_PAGE) { - await page.waitForNavigation(); - } - expect(page.url()).toBe(server.EMPTY_PAGE); - await browser.close(); + const options = Object.assign({}, defaultBrowserOptions); + options.args = [server.EMPTY_PAGE].concat(options.args || []); + const browser = await puppeteer.launch(options); + const pages = await browser.pages(); + expect(pages.length).toBe(1); + const page = pages[0]!; + if (page.url() !== server.EMPTY_PAGE) { + await page.waitForNavigation(); } - ); + expect(page.url()).toBe(server.EMPTY_PAGE); + await browser.close(); + }); it('should pass the timeout parameter to browser.waitForTarget', async () => { const {puppeteer, defaultBrowserOptions} = getTestState(); const options = Object.assign({}, defaultBrowserOptions, { @@ -615,24 +601,21 @@ describe('Launcher specs', function () { }); expect(error.message).toContain('either pipe or debugging port'); }); - itChromeOnly( - 'should launch Chrome properly with --no-startup-window and waitForInitialPage=false', - async () => { - const {defaultBrowserOptions, puppeteer} = getTestState(); - const options = { - waitForInitialPage: false, - // This is needed to prevent Puppeteer from adding an initial blank page. - // See also https://github.com/puppeteer/puppeteer/blob/ad6b736039436fcc5c0a262e5b575aa041427be3/src/node/Launcher.ts#L200 - ignoreDefaultArgs: true, - ...defaultBrowserOptions, - args: ['--no-startup-window'], - }; - const browser = await puppeteer.launch(options); - const pages = await browser.pages(); - expect(pages.length).toBe(0); - await browser.close(); - } - ); + it('should launch Chrome properly with --no-startup-window and waitForInitialPage=false', async () => { + const {defaultBrowserOptions, puppeteer} = getTestState(); + const options = { + waitForInitialPage: false, + // This is needed to prevent Puppeteer from adding an initial blank page. + // See also https://github.com/puppeteer/puppeteer/blob/ad6b736039436fcc5c0a262e5b575aa041427be3/src/node/Launcher.ts#L200 + ignoreDefaultArgs: true, + ...defaultBrowserOptions, + args: ['--no-startup-window'], + }; + const browser = await puppeteer.launch(options); + const pages = await browser.pages(); + expect(pages.length).toBe(0); + await browser.close(); + }); }); describe('Puppeteer.launch', function () { @@ -775,7 +758,7 @@ describe('Launcher specs', function () { }); // @see https://github.com/puppeteer/puppeteer/issues/4197 - itFailsFirefox('should support targetFilter option', async () => { + it('should support targetFilter option', async () => { const {server, puppeteer, defaultBrowserOptions} = getTestState(); const originalBrowser = await puppeteer.launch(defaultBrowserOptions); @@ -809,68 +792,62 @@ describe('Launcher specs', function () { .sort() ).toEqual(['about:blank', server.EMPTY_PAGE]); }); - itFailsFirefox( - 'should be able to reconnect to a disconnected browser', - async () => { - const {server, puppeteer, defaultBrowserOptions} = getTestState(); + it('should be able to reconnect to a disconnected browser', async () => { + const {server, puppeteer, defaultBrowserOptions} = getTestState(); - const originalBrowser = await puppeteer.launch(defaultBrowserOptions); - const browserWSEndpoint = originalBrowser.wsEndpoint(); - const page = await originalBrowser.newPage(); - await page.goto(server.PREFIX + '/frames/nested-frames.html'); - originalBrowser.disconnect(); + const originalBrowser = await puppeteer.launch(defaultBrowserOptions); + const browserWSEndpoint = originalBrowser.wsEndpoint(); + const page = await originalBrowser.newPage(); + await page.goto(server.PREFIX + '/frames/nested-frames.html'); + originalBrowser.disconnect(); - const browser = await puppeteer.connect({browserWSEndpoint}); - const pages = await browser.pages(); - const restoredPage = pages.find(page => { - return page.url() === server.PREFIX + '/frames/nested-frames.html'; - })!; - expect(utils.dumpFrames(restoredPage.mainFrame())).toEqual([ - 'http://localhost:/frames/nested-frames.html', - ' http://localhost:/frames/two-frames.html (2frames)', - ' http://localhost:/frames/frame.html (uno)', - ' http://localhost:/frames/frame.html (dos)', - ' http://localhost:/frames/frame.html (aframe)', - ]); - expect( - await restoredPage.evaluate(() => { - return 7 * 8; - }) - ).toBe(56); - await browser.close(); - } - ); + const browser = await puppeteer.connect({browserWSEndpoint}); + const pages = await browser.pages(); + const restoredPage = pages.find(page => { + return page.url() === server.PREFIX + '/frames/nested-frames.html'; + })!; + expect(utils.dumpFrames(restoredPage.mainFrame())).toEqual([ + 'http://localhost:/frames/nested-frames.html', + ' http://localhost:/frames/two-frames.html (2frames)', + ' http://localhost:/frames/frame.html (uno)', + ' http://localhost:/frames/frame.html (dos)', + ' http://localhost:/frames/frame.html (aframe)', + ]); + expect( + await restoredPage.evaluate(() => { + return 7 * 8; + }) + ).toBe(56); + await browser.close(); + }); // @see https://github.com/puppeteer/puppeteer/issues/4197#issuecomment-481793410 - itFailsFirefox( - 'should be able to connect to the same page simultaneously', - async () => { - const {puppeteer, defaultBrowserOptions} = getTestState(); + it('should be able to connect to the same page simultaneously', async () => { + const {puppeteer, defaultBrowserOptions} = getTestState(); - const browserOne = await puppeteer.launch(defaultBrowserOptions); - const browserTwo = await puppeteer.connect({ - browserWSEndpoint: browserOne.wsEndpoint(), - }); - const [page1, page2] = await Promise.all([ - new Promise(x => { - return browserOne.once('targetcreated', target => { - return x(target.page()); - }); - }), - browserTwo.newPage(), - ]); - expect( - await page1.evaluate(() => { - return 7 * 8; - }) - ).toBe(56); - expect( - await page2.evaluate(() => { - return 7 * 6; - }) - ).toBe(42); - await browserOne.close(); - } - ); + const browserOne = await puppeteer.launch(defaultBrowserOptions); + const browserTwo = await puppeteer.connect({ + browserWSEndpoint: browserOne.wsEndpoint(), + }); + const [page1, page2] = await Promise.all([ + new Promise(x => { + return browserOne.once('targetcreated', target => { + return x(target.page()); + }); + }), + browserTwo.newPage(), + ]); + expect( + await page1.evaluate(() => { + return 7 * 8; + }) + ).toBe(56); + expect( + await page2.evaluate(() => { + return 7 * 6; + }) + ).toBe(42); + await browserOne.close(); + }); it('should be able to reconnect', async () => { const {puppeteer, server, defaultBrowserOptions} = getTestState(); const browserOne = await puppeteer.launch(defaultBrowserOptions); @@ -933,7 +910,7 @@ describe('Launcher specs', function () { describe('when the product is chrome, platform is not darwin, and arch is arm64', () => { describe('and the executable exists', () => { - itChromeOnly('returns /usr/bin/chromium-browser', async () => { + it('returns /usr/bin/chromium-browser', async () => { const {puppeteer} = getTestState(); const osPlatformStub = sinon.stub(os, 'platform').returns('linux'); const osArchStub = sinon.stub(os, 'arch').returns('arm64'); @@ -972,33 +949,28 @@ describe('Launcher specs', function () { }); }); describe('and the executable does not exist', () => { - itChromeOnly( - 'does not return /usr/bin/chromium-browser', - async () => { - const {puppeteer} = getTestState(); - const osPlatformStub = sinon - .stub(os, 'platform') - .returns('linux'); - const osArchStub = sinon.stub(os, 'arch').returns('arm64'); - const fsExistsStub = sinon.stub(fs, 'existsSync'); - fsExistsStub.withArgs('/usr/bin/chromium-browser').returns(false); + it('does not return /usr/bin/chromium-browser', async () => { + const {puppeteer} = getTestState(); + const osPlatformStub = sinon.stub(os, 'platform').returns('linux'); + const osArchStub = sinon.stub(os, 'arch').returns('arm64'); + const fsExistsStub = sinon.stub(fs, 'existsSync'); + fsExistsStub.withArgs('/usr/bin/chromium-browser').returns(false); - const executablePath = puppeteer.executablePath(); + const executablePath = puppeteer.executablePath(); - expect(executablePath).not.toEqual('/usr/bin/chromium-browser'); + expect(executablePath).not.toEqual('/usr/bin/chromium-browser'); - osPlatformStub.restore(); - osArchStub.restore(); - fsExistsStub.restore(); - } - ); + osPlatformStub.restore(); + osArchStub.restore(); + fsExistsStub.restore(); + }); }); }); }); }); describe('Browser target events', function () { - itFailsFirefox('should work', async () => { + it('should work', async () => { const {server, puppeteer, defaultBrowserOptions} = getTestState(); const browser = await puppeteer.launch(defaultBrowserOptions); @@ -1021,51 +993,48 @@ describe('Launcher specs', function () { }); describe('Browser.Events.disconnected', function () { - itFailsFirefox( - 'should be emitted when: browser gets closed, disconnected or underlying websocket gets closed', - async () => { - const {puppeteer, defaultBrowserOptions} = getTestState(); - const originalBrowser = await puppeteer.launch(defaultBrowserOptions); - const browserWSEndpoint = originalBrowser.wsEndpoint(); - const remoteBrowser1 = await puppeteer.connect({ - browserWSEndpoint, - }); - const remoteBrowser2 = await puppeteer.connect({ - browserWSEndpoint, - }); + it('should be emitted when: browser gets closed, disconnected or underlying websocket gets closed', async () => { + const {puppeteer, defaultBrowserOptions} = getTestState(); + const originalBrowser = await puppeteer.launch(defaultBrowserOptions); + const browserWSEndpoint = originalBrowser.wsEndpoint(); + const remoteBrowser1 = await puppeteer.connect({ + browserWSEndpoint, + }); + const remoteBrowser2 = await puppeteer.connect({ + browserWSEndpoint, + }); - let disconnectedOriginal = 0; - let disconnectedRemote1 = 0; - let disconnectedRemote2 = 0; - originalBrowser.on('disconnected', () => { - return ++disconnectedOriginal; - }); - remoteBrowser1.on('disconnected', () => { - return ++disconnectedRemote1; - }); - remoteBrowser2.on('disconnected', () => { - return ++disconnectedRemote2; - }); + let disconnectedOriginal = 0; + let disconnectedRemote1 = 0; + let disconnectedRemote2 = 0; + originalBrowser.on('disconnected', () => { + return ++disconnectedOriginal; + }); + remoteBrowser1.on('disconnected', () => { + return ++disconnectedRemote1; + }); + remoteBrowser2.on('disconnected', () => { + return ++disconnectedRemote2; + }); - await Promise.all([ - utils.waitEvent(remoteBrowser2, 'disconnected'), - remoteBrowser2.disconnect(), - ]); + await Promise.all([ + utils.waitEvent(remoteBrowser2, 'disconnected'), + remoteBrowser2.disconnect(), + ]); - expect(disconnectedOriginal).toBe(0); - expect(disconnectedRemote1).toBe(0); - expect(disconnectedRemote2).toBe(1); + expect(disconnectedOriginal).toBe(0); + expect(disconnectedRemote1).toBe(0); + expect(disconnectedRemote2).toBe(1); - await Promise.all([ - utils.waitEvent(remoteBrowser1, 'disconnected'), - utils.waitEvent(originalBrowser, 'disconnected'), - originalBrowser.close(), - ]); + await Promise.all([ + utils.waitEvent(remoteBrowser1, 'disconnected'), + utils.waitEvent(originalBrowser, 'disconnected'), + originalBrowser.close(), + ]); - expect(disconnectedOriginal).toBe(1); - expect(disconnectedRemote1).toBe(1); - expect(disconnectedRemote2).toBe(1); - } - ); + expect(disconnectedOriginal).toBe(1); + expect(disconnectedRemote1).toBe(1); + expect(disconnectedRemote2).toBe(1); + }); }); }); diff --git a/test/src/mocha-utils.ts b/test/src/mocha-utils.ts index 709a67158f3..d21196c1e1a 100644 --- a/test/src/mocha-utils.ts +++ b/test/src/mocha-utils.ts @@ -17,7 +17,6 @@ import Protocol from 'devtools-protocol'; import expect from 'expect'; import * as fs from 'fs'; -import * as os from 'os'; import * as path from 'path'; import rimraf from 'rimraf'; import sinon from 'sinon'; @@ -34,6 +33,8 @@ import { import puppeteer from '../../lib/cjs/puppeteer/puppeteer.js'; import {TestServer} from '../../utils/testserver/lib/index.js'; import {extendExpectWithToBeGolden} from './utils.js'; +import * as Mocha from 'mocha'; +import {getTestId} from '../../utils/mochaRunner/lib/utils.js'; const setupServer = async () => { const assetsPath = path.join(__dirname, '../assets'); @@ -63,14 +64,14 @@ export const getTestState = (): PuppeteerTestState => { }; const product = - process.env['PRODUCT'] || process.env['PUPPETEER_PRODUCT'] || 'Chromium'; + process.env['PRODUCT'] || process.env['PUPPETEER_PRODUCT'] || 'chrome'; const alternativeInstall = process.env['PUPPETEER_ALT_INSTALL'] || false; const headless = (process.env['HEADLESS'] || 'true').trim().toLowerCase(); const isHeadless = headless === 'true' || headless === 'chrome'; const isFirefox = product === 'firefox'; -const isChrome = product === 'Chromium'; +const isChrome = product === 'chrome'; let extraLaunchOptions = {}; try { @@ -125,7 +126,11 @@ declare module 'expect/build/types' { } const setupGoldenAssertions = (): void => { - const suffix = product.toLowerCase(); + let suffix = product.toLowerCase(); + if (suffix === 'chrome') { + // TODO: to avoid moving golden folders. + suffix = 'chromium'; + } const GOLDEN_DIR = path.join(__dirname, `../golden-${suffix}`); const OUTPUT_DIR = path.join(__dirname, `../output-${suffix}`); if (fs.existsSync(OUTPUT_DIR)) { @@ -152,64 +157,9 @@ interface PuppeteerTestState { } const state: Partial = {}; -export const itFailsFirefox = ( - description: string, - body: Mocha.Func -): Mocha.Test => { - if (isFirefox) { - return xit(description, body); - } else { - return it(description, body); - } -}; - -export const itChromeOnly = ( - description: string, - body: Mocha.Func -): Mocha.Test => { - if (isChrome) { - return it(description, body); - } else { - return xit(description, body); - } -}; - -export const itHeadlessOnly = ( - description: string, - body: Mocha.Func -): Mocha.Test => { - if (isChrome && isHeadless === true) { - return it(description, body); - } else { - return xit(description, body); - } -}; - -export const itHeadfulOnly = ( - description: string, - body: Mocha.Func -): Mocha.Test => { - if (isChrome && isHeadless === false) { - return it(description, body); - } else { - return xit(description, body); - } -}; - -export const itFirefoxOnly = ( - description: string, - body: Mocha.Func -): Mocha.Test => { - if (isFirefox) { - return it(description, body); - } else { - return xit(description, body); - } -}; - export const itOnlyRegularInstall = ( description: string, - body: Mocha.Func + body: Mocha.AsyncFunc ): Mocha.Test => { if (alternativeInstall || process.env['BINARY']) { return xit(description, body); @@ -218,50 +168,10 @@ export const itOnlyRegularInstall = ( } }; -export const itFailsWindowsUntilDate = ( - date: Date, - description: string, - body: Mocha.Func -): Mocha.Test => { - if (os.platform() === 'win32' && Date.now() < date.getTime()) { - // we are within the deferred time so skip the test - return xit(description, body); - } - - return it(description, body); -}; - -export const itFailsWindows = ( - description: string, - body: Mocha.Func -): Mocha.Test => { - if (os.platform() === 'win32') { - return xit(description, body); - } - return it(description, body); -}; - -export const describeFailsFirefox = ( - description: string, - body: (this: Mocha.Suite) => void -): void | Mocha.Suite => { - if (isFirefox) { - return xdescribe(description, body); - } else { - return describe(description, body); - } -}; - -export const describeChromeOnly = ( - description: string, - body: (this: Mocha.Suite) => void -): Mocha.Suite | void => { - if (isChrome) { - return describe(description, body); - } -}; - -if (process.env['MOCHA_WORKER_ID'] === '0') { +if ( + process.env['MOCHA_WORKER_ID'] === undefined || + process.env['MOCHA_WORKER_ID'] === '0' +) { console.log( `Running unit tests with: -> product: ${product} @@ -290,7 +200,7 @@ export const setupTestBrowserHooks = (): void => { }); after(async () => { - await state.browser!.close(); + await state.browser?.close(); state.browser = undefined; }); }; @@ -302,7 +212,7 @@ export const setupTestPageAndContextHooks = (): void => { }); afterEach(async () => { - await state.context!.close(); + await state.context?.close(); state.context = undefined; state.page = undefined; }); @@ -387,3 +297,34 @@ export const shortWaitForArrayToHaveAtLeastNElements = async ( }); } }; + +type SyncFn = (this: Mocha.Context) => void; + +const skippedTests: string[] = process.env['PUPPETEER_SKIPPED_TEST_CONFIG'] + ? JSON.parse(process.env['PUPPETEER_SKIPPED_TEST_CONFIG']) + : []; + +function skipTestIfNeeded(test: Mocha.Test): void { + const testId = getTestId(test.file!, test.fullTitle()); + if ( + skippedTests.find(skippedTest => { + return testId.startsWith(skippedTest); + }) + ) { + try { + test.skip(); + } catch {} + } +} + +export function it(title: string, fn?: Mocha.AsyncFunc | SyncFn): Mocha.Test { + const test = Mocha.it.call(null, title, fn as any); + skipTestIfNeeded(test); + return test; +} + +it.only = function (title: string, fn?: Mocha.AsyncFunc | SyncFn): Mocha.Test { + const test = Mocha.it.only.call(null, title, fn as any); + skipTestIfNeeded(test); + return test; +}; diff --git a/test/src/mouse.spec.ts b/test/src/mouse.spec.ts index 078941e6b6f..661c2e8c2d5 100644 --- a/test/src/mouse.spec.ts +++ b/test/src/mouse.spec.ts @@ -19,9 +19,9 @@ import { getTestState, setupTestBrowserHooks, setupTestPageAndContextHooks, - itFailsFirefox, } from './mocha-utils.js'; import {KeyInput} from '../../lib/cjs/puppeteer/common/USKeyboardLayout.js'; +import {it} from './mocha-utils.js'; interface Dimensions { x: number; @@ -115,7 +115,7 @@ describe('Mouse', function () { }) ).toBe(text); }); - itFailsFirefox('should trigger hover state', async () => { + it('should trigger hover state', async () => { const {page, server} = getTestState(); await page.goto(server.PREFIX + '/input/scrollable.html'); @@ -138,24 +138,21 @@ describe('Mouse', function () { }) ).toBe('button-91'); }); - itFailsFirefox( - 'should trigger hover state with removed window.Node', - async () => { - const {page, server} = getTestState(); + it('should trigger hover state with removed window.Node', async () => { + const {page, server} = getTestState(); - await page.goto(server.PREFIX + '/input/scrollable.html'); + await page.goto(server.PREFIX + '/input/scrollable.html'); + await page.evaluate(() => { + // @ts-expect-error Expected. + return delete window.Node; + }); + await page.hover('#button-6'); + expect( await page.evaluate(() => { - // @ts-expect-error Expected. - return delete window.Node; - }); - await page.hover('#button-6'); - expect( - await page.evaluate(() => { - return document.querySelector('button:hover')!.id; - }) - ).toBe('button-6'); - } - ); + return document.querySelector('button:hover')!.id; + }) + ).toBe('button-6'); + }); it('should set modifier keys on click', async () => { const {page, server, isFirefox} = getTestState(); @@ -202,7 +199,7 @@ describe('Mouse', function () { } } }); - itFailsFirefox('should send mouse wheel events', async () => { + it('should send mouse wheel events', async () => { const {page, server} = getTestState(); await page.goto(server.PREFIX + '/input/wheel.html'); @@ -225,7 +222,7 @@ describe('Mouse', function () { height: 230, }); }); - itFailsFirefox('should tween mouse movement', async () => { + it('should tween mouse movement', async () => { const {page} = getTestState(); await page.mouse.move(100, 100); diff --git a/test/src/navigation.spec.ts b/test/src/navigation.spec.ts index e040f23de66..98efda2a9a5 100644 --- a/test/src/navigation.spec.ts +++ b/test/src/navigation.spec.ts @@ -20,12 +20,11 @@ import { getTestState, setupTestBrowserHooks, setupTestPageAndContextHooks, - itFailsFirefox, - describeFailsFirefox, } from './mocha-utils.js'; import os from 'os'; import {ServerResponse} from 'http'; import {HTTPRequest} from '../../lib/cjs/puppeteer/common/HTTPRequest.js'; +import {it} from './mocha-utils.js'; describe('navigation', function () { setupTestBrowserHooks(); @@ -67,7 +66,7 @@ describe('navigation', function () { const response = (await page.goto(server.PREFIX + '/historyapi.html'))!; expect(response.status()).toBe(200); }); - itFailsFirefox('should work with subframes return 204', async () => { + it('should work with subframes return 204', async () => { const {page, server} = getTestState(); server.setRoute('/frames/frame.html', (_req, res) => { @@ -82,7 +81,7 @@ describe('navigation', function () { }); expect(error).toBeUndefined(); }); - itFailsFirefox('should fail when server returns 204', async () => { + it('should fail when server returns 204', async () => { const {page, server, isChrome} = getTestState(); server.setRoute('/empty.html', (_req, res) => { @@ -124,29 +123,23 @@ describe('navigation', function () { const response = await page.goto(server.PREFIX + '/grid.html'); expect(response!.status()).toBe(200); }); - itFailsFirefox( - 'should navigate to empty page with networkidle0', - async () => { - const {page, server} = getTestState(); + it('should navigate to empty page with networkidle0', async () => { + const {page, server} = getTestState(); - const response = await page.goto(server.EMPTY_PAGE, { - waitUntil: 'networkidle0', - }); - expect(response!.status()).toBe(200); - } - ); - itFailsFirefox( - 'should navigate to empty page with networkidle2', - async () => { - const {page, server} = getTestState(); + const response = await page.goto(server.EMPTY_PAGE, { + waitUntil: 'networkidle0', + }); + expect(response!.status()).toBe(200); + }); + it('should navigate to empty page with networkidle2', async () => { + const {page, server} = getTestState(); - const response = await page.goto(server.EMPTY_PAGE, { - waitUntil: 'networkidle2', - }); - expect(response!.status()).toBe(200); - } - ); - itFailsFirefox('should fail when navigating to bad url', async () => { + const response = await page.goto(server.EMPTY_PAGE, { + waitUntil: 'networkidle2', + }); + expect(response!.status()).toBe(200); + }); + it('should fail when navigating to bad url', async () => { const {page, isChrome} = getTestState(); let error!: Error; @@ -175,7 +168,7 @@ describe('navigation', function () { : 'net::ERR_CERT_AUTHORITY_INVALID'; } - itFailsFirefox('should fail when navigating to bad SSL', async () => { + it('should fail when navigating to bad SSL', async () => { const {page, httpsServer, isChrome} = getTestState(); // Make sure that network events do not emit 'undefined'. @@ -311,7 +304,7 @@ describe('navigation', function () { const response = (await page.goto(server.EMPTY_PAGE))!; expect(response.ok()).toBe(true); }); - itFailsFirefox('should work when navigating to data url', async () => { + it('should work when navigating to data url', async () => { const {page} = getTestState(); const response = (await page.goto('data:text/html,hello'))!; @@ -334,85 +327,79 @@ describe('navigation', function () { expect(response.ok()).toBe(true); expect(response.url()).toBe(server.EMPTY_PAGE); }); - itFailsFirefox( - 'should wait for network idle to succeed navigation', - async () => { - const {page, server} = getTestState(); + it('should wait for network idle to succeed navigation', async () => { + const {page, server} = getTestState(); - let responses: ServerResponse[] = []; - // Hold on to a bunch of requests without answering. - server.setRoute('/fetch-request-a.js', (_req, res) => { - return responses.push(res); - }); - server.setRoute('/fetch-request-b.js', (_req, res) => { - return responses.push(res); - }); - server.setRoute('/fetch-request-c.js', (_req, res) => { - return responses.push(res); - }); - server.setRoute('/fetch-request-d.js', (_req, res) => { - return responses.push(res); - }); - const initialFetchResourcesRequested = Promise.all([ - server.waitForRequest('/fetch-request-a.js'), - server.waitForRequest('/fetch-request-b.js'), - server.waitForRequest('/fetch-request-c.js'), - ]); - const secondFetchResourceRequested = server.waitForRequest( - '/fetch-request-d.js' - ); + let responses: ServerResponse[] = []; + // Hold on to a bunch of requests without answering. + server.setRoute('/fetch-request-a.js', (_req, res) => { + return responses.push(res); + }); + server.setRoute('/fetch-request-b.js', (_req, res) => { + return responses.push(res); + }); + server.setRoute('/fetch-request-c.js', (_req, res) => { + return responses.push(res); + }); + server.setRoute('/fetch-request-d.js', (_req, res) => { + return responses.push(res); + }); + const initialFetchResourcesRequested = Promise.all([ + server.waitForRequest('/fetch-request-a.js'), + server.waitForRequest('/fetch-request-b.js'), + server.waitForRequest('/fetch-request-c.js'), + ]); + const secondFetchResourceRequested = server.waitForRequest( + '/fetch-request-d.js' + ); - // Navigate to a page which loads immediately and then does a bunch of - // requests via javascript's fetch method. - const navigationPromise = page.goto( - server.PREFIX + '/networkidle.html', - { - waitUntil: 'networkidle0', - } - ); - // Track when the navigation gets completed. - let navigationFinished = false; - navigationPromise.then(() => { - return (navigationFinished = true); - }); + // Navigate to a page which loads immediately and then does a bunch of + // requests via javascript's fetch method. + const navigationPromise = page.goto(server.PREFIX + '/networkidle.html', { + waitUntil: 'networkidle0', + }); + // Track when the navigation gets completed. + let navigationFinished = false; + navigationPromise.then(() => { + return (navigationFinished = true); + }); - // Wait for the page's 'load' event. - await new Promise(fulfill => { - return page.once('load', fulfill); - }); - expect(navigationFinished).toBe(false); + // Wait for the page's 'load' event. + await new Promise(fulfill => { + return page.once('load', fulfill); + }); + expect(navigationFinished).toBe(false); - // Wait for the initial three resources to be requested. - await initialFetchResourcesRequested; + // Wait for the initial three resources to be requested. + await initialFetchResourcesRequested; - // Expect navigation still to be not finished. - expect(navigationFinished).toBe(false); + // Expect navigation still to be not finished. + expect(navigationFinished).toBe(false); - // Respond to initial requests. - for (const response of responses) { - response.statusCode = 404; - response.end(`File not found`); - } - - // Reset responses array - responses = []; - - // Wait for the second round to be requested. - await secondFetchResourceRequested; - // Expect navigation still to be not finished. - expect(navigationFinished).toBe(false); - - // Respond to requests. - for (const response of responses) { - response.statusCode = 404; - response.end(`File not found`); - } - - const response = (await navigationPromise)!; - // Expect navigation to succeed. - expect(response.ok()).toBe(true); + // Respond to initial requests. + for (const response of responses) { + response.statusCode = 404; + response.end(`File not found`); } - ); + + // Reset responses array + responses = []; + + // Wait for the second round to be requested. + await secondFetchResourceRequested; + // Expect navigation still to be not finished. + expect(navigationFinished).toBe(false); + + // Respond to requests. + for (const response of responses) { + response.statusCode = 404; + response.end(`File not found`); + } + + const response = (await navigationPromise)!; + // Expect navigation to succeed. + expect(response.ok()).toBe(true); + }); it('should not leak listeners during navigation', async () => { const {page, server} = getTestState(); @@ -461,38 +448,32 @@ describe('navigation', function () { process.removeListener('warning', warningHandler); expect(warning).toBe(null); }); - itFailsFirefox( - 'should navigate to dataURL and fire dataURL requests', - async () => { - const {page} = getTestState(); + it('should navigate to dataURL and fire dataURL requests', async () => { + const {page} = getTestState(); - const requests: HTTPRequest[] = []; - page.on('request', request => { - return !utils.isFavicon(request) && requests.push(request); - }); - const dataURL = 'data:text/html,
yo
'; - const response = (await page.goto(dataURL))!; - expect(response.status()).toBe(200); - expect(requests.length).toBe(1); - expect(requests[0]!.url()).toBe(dataURL); - } - ); - itFailsFirefox( - 'should navigate to URL with hash and fire requests without hash', - async () => { - const {page, server} = getTestState(); + const requests: HTTPRequest[] = []; + page.on('request', request => { + return !utils.isFavicon(request) && requests.push(request); + }); + const dataURL = 'data:text/html,
yo
'; + const response = (await page.goto(dataURL))!; + expect(response.status()).toBe(200); + expect(requests.length).toBe(1); + expect(requests[0]!.url()).toBe(dataURL); + }); + it('should navigate to URL with hash and fire requests without hash', async () => { + const {page, server} = getTestState(); - const requests: HTTPRequest[] = []; - page.on('request', request => { - return !utils.isFavicon(request) && requests.push(request); - }); - const response = (await page.goto(server.EMPTY_PAGE + '#hash'))!; - expect(response.status()).toBe(200); - expect(response.url()).toBe(server.EMPTY_PAGE); - expect(requests.length).toBe(1); - expect(requests[0]!.url()).toBe(server.EMPTY_PAGE); - } - ); + const requests: HTTPRequest[] = []; + page.on('request', request => { + return !utils.isFavicon(request) && requests.push(request); + }); + const response = (await page.goto(server.EMPTY_PAGE + '#hash'))!; + expect(response.status()).toBe(200); + expect(response.url()).toBe(server.EMPTY_PAGE); + expect(requests.length).toBe(1); + expect(requests[0]!.url()).toBe(server.EMPTY_PAGE); + }); it('should work with self requesting page', async () => { const {page, server} = getTestState(); @@ -512,7 +493,7 @@ describe('navigation', function () { } expect(error.message).toContain(url); }); - itFailsFirefox('should send referer', async () => { + it('should send referer', async () => { const {page, server} = getTestState(); const [request1, request2] = await Promise.all([ @@ -582,7 +563,7 @@ describe('navigation', function () { expect(response).toBe(null); expect(page.url()).toBe(server.EMPTY_PAGE + '#foobar'); }); - itFailsFirefox('should work with history.pushState()', async () => { + it('should work with history.pushState()', async () => { const {page, server} = getTestState(); await page.goto(server.EMPTY_PAGE); @@ -599,7 +580,7 @@ describe('navigation', function () { expect(response).toBe(null); expect(page.url()).toBe(server.PREFIX + '/wow.html'); }); - itFailsFirefox('should work with history.replaceState()', async () => { + it('should work with history.replaceState()', async () => { const {page, server} = getTestState(); await page.goto(server.EMPTY_PAGE); @@ -616,13 +597,11 @@ describe('navigation', function () { expect(response).toBe(null); expect(page.url()).toBe(server.PREFIX + '/replaced.html'); }); - itFailsFirefox( - 'should work with DOM history.back()/history.forward()', - async () => { - const {page, server} = getTestState(); + it('should work with DOM history.back()/history.forward()', async () => { + const {page, server} = getTestState(); - await page.goto(server.EMPTY_PAGE); - await page.setContent(` + await page.goto(server.EMPTY_PAGE); + await page.setContent(` back forward `); - expect(page.url()).toBe(server.PREFIX + '/second.html'); - const [backResponse] = await Promise.all([ - page.waitForNavigation(), - page.click('a#back'), - ]); - expect(backResponse).toBe(null); - expect(page.url()).toBe(server.PREFIX + '/first.html'); - const [forwardResponse] = await Promise.all([ - page.waitForNavigation(), - page.click('a#forward'), - ]); - expect(forwardResponse).toBe(null); - expect(page.url()).toBe(server.PREFIX + '/second.html'); - } - ); - itFailsFirefox( - 'should work when subframe issues window.stop()', - async () => { - const {page, server} = getTestState(); + expect(page.url()).toBe(server.PREFIX + '/second.html'); + const [backResponse] = await Promise.all([ + page.waitForNavigation(), + page.click('a#back'), + ]); + expect(backResponse).toBe(null); + expect(page.url()).toBe(server.PREFIX + '/first.html'); + const [forwardResponse] = await Promise.all([ + page.waitForNavigation(), + page.click('a#forward'), + ]); + expect(forwardResponse).toBe(null); + expect(page.url()).toBe(server.PREFIX + '/second.html'); + }); + it('should work when subframe issues window.stop()', async () => { + const {page, server} = getTestState(); - server.setRoute('/frames/style.css', () => {}); - const navigationPromise = page.goto( - server.PREFIX + '/frames/one-frame.html' - ); - const frame = await utils.waitEvent(page, 'frameattached'); - await new Promise(fulfill => { - page.on('framenavigated', f => { - if (f === frame) { - fulfill(); - } - }); + server.setRoute('/frames/style.css', () => {}); + const navigationPromise = page.goto( + server.PREFIX + '/frames/one-frame.html' + ); + const frame = await utils.waitEvent(page, 'frameattached'); + await new Promise(fulfill => { + page.on('framenavigated', f => { + if (f === frame) { + fulfill(); + } }); - await Promise.all([ - frame.evaluate(() => { - return window.stop(); - }), - navigationPromise, - ]); - } - ); + }); + await Promise.all([ + frame.evaluate(() => { + return window.stop(); + }), + navigationPromise, + ]); + }); }); describe('Page.goBack', function () { @@ -692,7 +667,7 @@ describe('navigation', function () { response = (await page.goForward())!; expect(response).toBe(null); }); - itFailsFirefox('should work with HistoryAPI', async () => { + it('should work with HistoryAPI', async () => { const {page, server} = getTestState(); await page.goto(server.EMPTY_PAGE); @@ -711,7 +686,7 @@ describe('navigation', function () { }); }); - describeFailsFirefox('Frame.goto', function () { + describe('Frame.goto', function () { it('should navigate subframes', async () => { const {page, server} = getTestState(); @@ -776,7 +751,7 @@ describe('navigation', function () { }); }); - describeFailsFirefox('Frame.waitForNavigation', function () { + describe('Frame.waitForNavigation', function () { it('should work', async () => { const {page, server} = getTestState(); diff --git a/test/src/network.spec.ts b/test/src/network.spec.ts index 9faad3bf970..b71b0187f75 100644 --- a/test/src/network.spec.ts +++ b/test/src/network.spec.ts @@ -22,14 +22,11 @@ import { getTestState, setupTestBrowserHooks, setupTestPageAndContextHooks, - itFailsFirefox, - describeFailsFirefox, - itChromeOnly, - itFirefoxOnly, } from './mocha-utils.js'; import {HTTPRequest} from '../../lib/cjs/puppeteer/common/HTTPRequest.js'; import {HTTPResponse} from '../../lib/cjs/puppeteer/common/HTTPResponse.js'; import {ServerResponse} from 'http'; +import {it} from './mocha-utils.js'; describe('network', function () { setupTestBrowserHooks(); @@ -83,7 +80,7 @@ describe('network', function () { expect(requests.length).toBe(1); expect(requests[0]!.frame()).toBe(page.mainFrame()); }); - itFailsFirefox('should work for subframe navigation request', async () => { + it('should work for subframe navigation request', async () => { const {page, server} = getTestState(); (await page.goto(server.EMPTY_PAGE))!; @@ -115,13 +112,13 @@ describe('network', function () { }); describe('Request.headers', function () { - itChromeOnly('should define Chrome as user agent header', async () => { + it('should define Chrome as user agent header', async () => { const {page, server} = getTestState(); const response = (await page.goto(server.EMPTY_PAGE))!; expect(response.request().headers()['user-agent']).toContain('Chrome'); }); - itFirefoxOnly('should define Firefox as user agent header', async () => { + it('should define Firefox as user agent header', async () => { const {page, server} = getTestState(); const response = (await page.goto(server.EMPTY_PAGE))!; @@ -142,7 +139,7 @@ describe('network', function () { }); }); - describeFailsFirefox('Request.initiator', () => { + describe('Request.initiator', () => { it('should return the initiator', async () => { const {page, server} = getTestState(); @@ -187,7 +184,7 @@ describe('network', function () { }); }); - describeFailsFirefox('Response.fromCache', function () { + describe('Response.fromCache', function () { it('should return |false| for non-cached content', async () => { const {page, server} = getTestState(); @@ -218,7 +215,7 @@ describe('network', function () { }); }); - describeFailsFirefox('Response.fromServiceWorker', function () { + describe('Response.fromServiceWorker', function () { it('should return |false| for non-service-worker content', async () => { const {page, server} = getTestState(); @@ -253,7 +250,7 @@ describe('network', function () { }); }); - describeFailsFirefox('Request.postData', function () { + describe('Request.postData', function () { it('should work', async () => { const {page, server} = getTestState(); @@ -284,7 +281,7 @@ describe('network', function () { }); }); - describeFailsFirefox('Response.text', function () { + describe('Response.text', function () { it('should work', async () => { const {page, server} = getTestState(); @@ -367,7 +364,7 @@ describe('network', function () { }); }); - describeFailsFirefox('Response.json', function () { + describe('Response.json', function () { it('should work', async () => { const {page, server} = getTestState(); @@ -376,7 +373,7 @@ describe('network', function () { }); }); - describeFailsFirefox('Response.buffer', function () { + describe('Response.buffer', function () { it('should work', async () => { const {page, server} = getTestState(); @@ -461,7 +458,7 @@ describe('network', function () { }); }); - describeFailsFirefox('Response.timing', function () { + describe('Response.timing', function () { it('returns timing information', async () => { const {page, server} = getTestState(); const responses: HTTPResponse[] = []; @@ -474,7 +471,7 @@ describe('network', function () { }); }); - describeFailsFirefox('Network Events', function () { + describe('Network Events', function () { it('Page.Events.Request', async () => { const {page, server} = getTestState(); @@ -624,7 +621,7 @@ describe('network', function () { }); describe('Request.isNavigationRequest', () => { - itFailsFirefox('should work', async () => { + it('should work', async () => { const {page, server} = getTestState(); const requests = new Map(); @@ -639,7 +636,7 @@ describe('network', function () { expect(requests.get('script.js').isNavigationRequest()).toBe(false); expect(requests.get('style.css').isNavigationRequest()).toBe(false); }); - itFailsFirefox('should work with request interception', async () => { + it('should work with request interception', async () => { const {page, server} = getTestState(); const requests = new Map(); @@ -656,7 +653,7 @@ describe('network', function () { expect(requests.get('script.js').isNavigationRequest()).toBe(false); expect(requests.get('style.css').isNavigationRequest()).toBe(false); }); - itFailsFirefox('should work when navigating to image', async () => { + it('should work when navigating to image', async () => { const {page, server} = getTestState(); const requests: HTTPRequest[] = []; @@ -668,7 +665,7 @@ describe('network', function () { }); }); - describeFailsFirefox('Page.setExtraHTTPHeaders', function () { + describe('Page.setExtraHTTPHeaders', function () { it('should work', async () => { const {page, server} = getTestState(); @@ -697,7 +694,7 @@ describe('network', function () { }); }); - describeFailsFirefox('Page.authenticate', function () { + describe('Page.authenticate', function () { it('should work', async () => { const {page, server} = getTestState(); @@ -794,7 +791,7 @@ describe('network', function () { }); }); - describeFailsFirefox('raw network headers', async () => { + describe('raw network headers', async () => { it('Same-origin set-cookie navigation', async () => { const {page, server} = getTestState(); diff --git a/test/src/oopif.spec.ts b/test/src/oopif.spec.ts index 17762636d23..174d3c5ae69 100644 --- a/test/src/oopif.spec.ts +++ b/test/src/oopif.spec.ts @@ -16,18 +16,15 @@ import utils from './utils.js'; import expect from 'expect'; -import { - getTestState, - describeChromeOnly, - itFailsFirefox, -} from './mocha-utils.js'; +import {getTestState} from './mocha-utils.js'; import { Browser, BrowserContext, } from '../../lib/cjs/puppeteer/common/Browser.js'; import {Page} from '../../lib/cjs/puppeteer/common/Page.js'; +import {it} from './mocha-utils.js'; -describeChromeOnly('OOPIF', function () { +describe('OOPIF', function () { /* We use a special browser for this test as we need the --site-per-process flag */ let browser: Browser; let context: BrowserContext; @@ -207,6 +204,7 @@ describeChromeOnly('OOPIF', function () { await utils.navigateFrame(page, 'frame1', server.EMPTY_PAGE); expect(frame.url()).toBe(server.EMPTY_PAGE); }); + it('should support evaluating in oop iframes', async () => { const {server} = getTestState(); @@ -420,7 +418,7 @@ describeChromeOnly('OOPIF', function () { browser1.disconnect(); }); - itFailsFirefox('should support lazy OOP frames', async () => { + it('should support lazy OOP frames', async () => { const {server} = getTestState(); await page.goto(server.PREFIX + '/lazy-oopif-frame.html'); diff --git a/test/src/page.spec.ts b/test/src/page.spec.ts index 1f7510e4648..3d59994a12a 100644 --- a/test/src/page.spec.ts +++ b/test/src/page.spec.ts @@ -22,13 +22,12 @@ import {CDPSession} from '../../lib/cjs/puppeteer/common/Connection.js'; import {ConsoleMessage} from '../../lib/cjs/puppeteer/common/ConsoleMessage.js'; import {Metrics, Page} from '../../lib/cjs/puppeteer/common/Page.js'; import { - describeFailsFirefox, getTestState, - itFailsFirefox, setupTestBrowserHooks, setupTestPageAndContextHooks, } from './mocha-utils.js'; import utils, {attachFrame, waitEvent} from './utils.js'; +import {it} from './mocha-utils.js'; describe('Page', function () { setupTestBrowserHooks(); @@ -59,7 +58,7 @@ describe('Page', function () { await newPage.close(); expect(await browser.pages()).not.toContain(newPage); }); - itFailsFirefox('should run beforeunload if asked for', async () => { + it('should run beforeunload if asked for', async () => { const {context, server, isChrome} = getTestState(); const newPage = await context.newPage(); @@ -79,7 +78,7 @@ describe('Page', function () { await dialog.accept(); await pageClosingPromise; }); - itFailsFirefox('should *not* run beforeunload by default', async () => { + it('should *not* run beforeunload by default', async () => { const {context, server} = getTestState(); const newPage = await context.newPage(); @@ -97,7 +96,7 @@ describe('Page', function () { await newPage.close(); expect(newPage.isClosed()).toBe(true); }); - itFailsFirefox('should terminate network waiters', async () => { + it('should terminate network waiters', async () => { const {context, server} = getTestState(); const newPage = await context.newPage(); @@ -182,7 +181,7 @@ describe('Page', function () { }); }); - describeFailsFirefox('Page.Events.error', function () { + describe('Page.Events.error', function () { it('should throw when page crashes', async () => { const {page} = getTestState(); @@ -196,7 +195,7 @@ describe('Page', function () { }); }); - describeFailsFirefox('Page.Events.Popup', function () { + describe('Page.Events.Popup', function () { it('should work', async () => { const {page} = getTestState(); @@ -354,7 +353,7 @@ describe('Page', function () { await page.goto(server.EMPTY_PAGE); expect(await getPermission(page, 'geolocation')).toBe('prompt'); }); - itFailsFirefox('should deny permission when not listed', async () => { + it('should deny permission when not listed', async () => { const {page, server, context} = getTestState(); await page.goto(server.EMPTY_PAGE); @@ -374,14 +373,14 @@ describe('Page', function () { }); expect(error.message).toBe('Unknown permission: foo'); }); - itFailsFirefox('should grant permission when listed', async () => { + it('should grant permission when listed', async () => { const {page, server, context} = getTestState(); await page.goto(server.EMPTY_PAGE); await context.overridePermissions(server.EMPTY_PAGE, ['geolocation']); expect(await getPermission(page, 'geolocation')).toBe('granted'); }); - itFailsFirefox('should reset permissions', async () => { + it('should reset permissions', async () => { const {page, server, context} = getTestState(); await page.goto(server.EMPTY_PAGE); @@ -390,7 +389,7 @@ describe('Page', function () { await context.clearPermissionOverrides(); expect(await getPermission(page, 'geolocation')).toBe('prompt'); }); - itFailsFirefox('should trigger permission onchange', async () => { + it('should trigger permission onchange', async () => { const {page, server, context, isHeadless} = getTestState(); // TODO: re-enable this test in headful once crbug.com/1324480 rolls out. @@ -434,33 +433,30 @@ describe('Page', function () { }) ).toEqual(['prompt', 'denied', 'granted', 'prompt']); }); - itFailsFirefox( - 'should isolate permissions between browser contexts', - async () => { - const {page, server, context, browser} = getTestState(); + it('should isolate permissions between browser contexts', async () => { + const {page, server, context, browser} = getTestState(); - await page.goto(server.EMPTY_PAGE); - const otherContext = await browser.createIncognitoBrowserContext(); - const otherPage = await otherContext.newPage(); - await otherPage.goto(server.EMPTY_PAGE); - expect(await getPermission(page, 'geolocation')).toBe('prompt'); - expect(await getPermission(otherPage, 'geolocation')).toBe('prompt'); + await page.goto(server.EMPTY_PAGE); + const otherContext = await browser.createIncognitoBrowserContext(); + const otherPage = await otherContext.newPage(); + await otherPage.goto(server.EMPTY_PAGE); + expect(await getPermission(page, 'geolocation')).toBe('prompt'); + expect(await getPermission(otherPage, 'geolocation')).toBe('prompt'); - await context.overridePermissions(server.EMPTY_PAGE, []); - await otherContext.overridePermissions(server.EMPTY_PAGE, [ - 'geolocation', - ]); - expect(await getPermission(page, 'geolocation')).toBe('denied'); - expect(await getPermission(otherPage, 'geolocation')).toBe('granted'); + await context.overridePermissions(server.EMPTY_PAGE, []); + await otherContext.overridePermissions(server.EMPTY_PAGE, [ + 'geolocation', + ]); + expect(await getPermission(page, 'geolocation')).toBe('denied'); + expect(await getPermission(otherPage, 'geolocation')).toBe('granted'); - await context.clearPermissionOverrides(); - expect(await getPermission(page, 'geolocation')).toBe('prompt'); - expect(await getPermission(otherPage, 'geolocation')).toBe('granted'); + await context.clearPermissionOverrides(); + expect(await getPermission(page, 'geolocation')).toBe('prompt'); + expect(await getPermission(otherPage, 'geolocation')).toBe('granted'); - await otherContext.close(); - } - ); - itFailsFirefox('should grant persistent-storage', async () => { + await otherContext.close(); + }); + it('should grant persistent-storage', async () => { const {page, server, context} = getTestState(); await page.goto(server.EMPTY_PAGE); @@ -475,7 +471,7 @@ describe('Page', function () { }); describe('Page.setGeolocation', function () { - itFailsFirefox('should work', async () => { + it('should work', async () => { const {page, server, context} = getTestState(); await context.overridePermissions(server.PREFIX, ['geolocation']); @@ -509,7 +505,7 @@ describe('Page', function () { }); }); - describeFailsFirefox('Page.setOfflineMode', function () { + describe('Page.setOfflineMode', function () { it('should work', async () => { const {page, server} = getTestState(); @@ -547,7 +543,7 @@ describe('Page', function () { }); describe('ExecutionContext.queryObjects', function () { - itFailsFirefox('should work', async () => { + it('should work', async () => { const {page} = getTestState(); // Instantiate an object @@ -567,7 +563,7 @@ describe('Page', function () { }, objectsHandle); expect(values).toEqual(['hello', 'world']); }); - itFailsFirefox('should work for non-blank page', async () => { + it('should work for non-blank page', async () => { const {page, server} = getTestState(); // Instantiate an object @@ -613,7 +609,7 @@ describe('Page', function () { }); }); - describeFailsFirefox('Page.Events.Console', function () { + describe('Page.Events.Console', function () { it('should work', async () => { const {page} = getTestState(); @@ -802,7 +798,7 @@ describe('Page', function () { }); }); - describeFailsFirefox('Page.metrics', function () { + describe('Page.metrics', function () { it('should get metrics from a page', async () => { const {page} = getTestState(); @@ -1108,7 +1104,7 @@ describe('Page', function () { }); }); - describeFailsFirefox('Page.exposeFunction', function () { + describe('Page.exposeFunction', function () { it('should work', async () => { const {page} = getTestState(); @@ -1266,7 +1262,7 @@ describe('Page', function () { }); }); - describeFailsFirefox('Page.Events.PageError', function () { + describe('Page.Events.PageError', function () { it('should fire', async () => { const {page, server} = getTestState(); @@ -1329,7 +1325,7 @@ describe('Page', function () { }) ).toContain('iPhone'); }); - itFailsFirefox('should work with additional userAgentMetdata', async () => { + it('should work with additional userAgentMetdata', async () => { const {page, server} = getTestState(); await page.setUserAgent('MockBrowser', { @@ -1496,7 +1492,7 @@ describe('Page', function () { }); }); - describeFailsFirefox('Page.setBypassCSP', function () { + describe('Page.setBypassCSP', function () { it('should bypass CSP meta tag', async () => { const {page, server} = getTestState(); @@ -1879,21 +1875,18 @@ describe('Page', function () { ).toBe('rgb(0, 128, 0)'); }); - itFailsFirefox( - 'should throw when added with content to the CSP page', - async () => { - const {page, server} = getTestState(); + it('should throw when added with content to the CSP page', async () => { + const {page, server} = getTestState(); - await page.goto(server.PREFIX + '/csp.html'); - let error!: Error; - await page - .addStyleTag({content: 'body { background-color: green; }'}) - .catch(error_ => { - return (error = error_); - }); - expect(error).toBeTruthy(); - } - ); + await page.goto(server.PREFIX + '/csp.html'); + let error!: Error; + await page + .addStyleTag({content: 'body { background-color: green; }'}) + .catch(error_ => { + return (error = error_); + }); + expect(error).toBeTruthy(); + }); it('should throw when added with URL to the CSP page', async () => { const {page, server} = getTestState(); @@ -1921,7 +1914,7 @@ describe('Page', function () { }); }); - describeFailsFirefox('Page.setJavaScriptEnabled', function () { + describe('Page.setJavaScriptEnabled', function () { it('should work', async () => { const {page} = getTestState(); @@ -1962,23 +1955,20 @@ describe('Page', function () { ]); expect(nonCachedRequest.headers['if-modified-since']).toBe(undefined); }); - itFailsFirefox( - 'should stay disabled when toggling request interception on/off', - async () => { - const {page, server} = getTestState(); + it('should stay disabled when toggling request interception on/off', async () => { + const {page, server} = getTestState(); - await page.setCacheEnabled(false); - await page.setRequestInterception(true); - await page.setRequestInterception(false); + await page.setCacheEnabled(false); + await page.setRequestInterception(true); + await page.setRequestInterception(false); - await page.goto(server.PREFIX + '/cached/one-style.html'); - const [nonCachedRequest] = await Promise.all([ - server.waitForRequest('/cached/one-style.html'), - page.reload(), - ]); - expect(nonCachedRequest.headers['if-modified-since']).toBe(undefined); - } - ); + await page.goto(server.PREFIX + '/cached/one-style.html'); + const [nonCachedRequest] = await Promise.all([ + server.waitForRequest('/cached/one-style.html'), + page.reload(), + ]); + expect(nonCachedRequest.headers['if-modified-since']).toBe(undefined); + }); }); describe('printing to PDF', function () { @@ -2220,33 +2210,30 @@ describe('Page', function () { expect(error.message).toContain('Values must be strings'); }); // @see https://github.com/puppeteer/puppeteer/issues/3327 - itFailsFirefox( - 'should work when re-defining top-level Event class', - async () => { - const {page, server} = getTestState(); + it('should work when re-defining top-level Event class', async () => { + const {page, server} = getTestState(); - await page.goto(server.PREFIX + '/input/select.html'); + await page.goto(server.PREFIX + '/input/select.html'); + await page.evaluate(() => { + // @ts-expect-error Expected. + return (window.Event = undefined); + }); + await page.select('select', 'blue'); + expect( await page.evaluate(() => { - // @ts-expect-error Expected. - return (window.Event = undefined); - }); - await page.select('select', 'blue'); - expect( - await page.evaluate(() => { - return (globalThis as any).result.onInput; - }) - ).toEqual(['blue']); - expect( - await page.evaluate(() => { - return (globalThis as any).result.onChange; - }) - ).toEqual(['blue']); - } - ); + return (globalThis as any).result.onInput; + }) + ).toEqual(['blue']); + expect( + await page.evaluate(() => { + return (globalThis as any).result.onChange; + }) + ).toEqual(['blue']); + }); }); describe('Page.Events.Close', function () { - itFailsFirefox('should work with window.close', async () => { + it('should work with window.close', async () => { const {page, context} = getTestState(); const newPagePromise = new Promise(fulfill => { diff --git a/test/src/proxy.spec.ts b/test/src/proxy.spec.ts index eb71905355d..9aac24e4bcb 100644 --- a/test/src/proxy.spec.ts +++ b/test/src/proxy.spec.ts @@ -17,15 +17,12 @@ import expect from 'expect'; import http from 'http'; import os from 'os'; -import { - getTestState, - describeFailsFirefox, - itFailsWindows, -} from './mocha-utils.js'; +import {getTestState} from './mocha-utils.js'; import type {Server, IncomingMessage, ServerResponse} from 'http'; import type {Browser} from '../../lib/cjs/puppeteer/common/Browser.js'; import type {AddressInfo} from 'net'; import {TestServer} from '../../utils/testserver/lib/index.js'; +import {it} from './mocha-utils.js'; let HOSTNAME = os.hostname(); @@ -53,7 +50,7 @@ function getEmptyPageUrl(server: TestServer): string { return `http://${HOSTNAME}:${server.PORT}${emptyPagePath}`; } -describeFailsFirefox('request proxy', () => { +describe('request proxy', () => { let browser: Browser; let proxiedRequestUrls: string[]; let proxyServer: Server; @@ -194,28 +191,25 @@ describeFailsFirefox('request proxy', () => { /** * See issues #7873, #7719, and #7698. */ - itFailsWindows( - 'should proxy requests when configured at context level', - async () => { - const {puppeteer, defaultBrowserOptions, server} = getTestState(); - const emptyPageUrl = getEmptyPageUrl(server); + it('should proxy requests when configured at context level', async () => { + const {puppeteer, defaultBrowserOptions, server} = getTestState(); + const emptyPageUrl = getEmptyPageUrl(server); - browser = await puppeteer.launch({ - ...defaultBrowserOptions, - args: defaultArgs, - }); + browser = await puppeteer.launch({ + ...defaultBrowserOptions, + args: defaultArgs, + }); - const context = await browser.createIncognitoBrowserContext({ - proxyServer: proxyServerUrl, - }); - const page = await context.newPage(); - const response = (await page.goto(emptyPageUrl))!; + const context = await browser.createIncognitoBrowserContext({ + proxyServer: proxyServerUrl, + }); + const page = await context.newPage(); + const response = (await page.goto(emptyPageUrl))!; - expect(response.ok()).toBe(true); + expect(response.ok()).toBe(true); - expect(proxiedRequestUrls).toEqual([emptyPageUrl]); - } - ); + expect(proxiedRequestUrls).toEqual([emptyPageUrl]); + }); it('should respect proxy bypass list when configured at context level', async () => { const {puppeteer, defaultBrowserOptions, server} = getTestState(); diff --git a/test/src/queryhandler.spec.ts b/test/src/queryhandler.spec.ts index 30e5d69254d..921fbb7d6b2 100644 --- a/test/src/queryhandler.spec.ts +++ b/test/src/queryhandler.spec.ts @@ -20,6 +20,7 @@ import { setupTestBrowserHooks, setupTestPageAndContextHooks, } from './mocha-utils.js'; +import {it} from './mocha-utils.js'; describe('Query handler tests', function () { setupTestBrowserHooks(); diff --git a/test/src/queryselector.spec.ts b/test/src/queryselector.spec.ts index 7eeab8e44e7..8c94a60e74c 100644 --- a/test/src/queryselector.spec.ts +++ b/test/src/queryselector.spec.ts @@ -20,6 +20,7 @@ import { setupTestBrowserHooks, setupTestPageAndContextHooks, } from './mocha-utils.js'; +import {it} from './mocha-utils.js'; describe('querySelector', function () { setupTestBrowserHooks(); diff --git a/test/src/requestinterception-experimental.spec.ts b/test/src/requestinterception-experimental.spec.ts index 687588237d5..02971d20234 100644 --- a/test/src/requestinterception-experimental.spec.ts +++ b/test/src/requestinterception-experimental.spec.ts @@ -22,7 +22,6 @@ import { getTestState, setupTestBrowserHooks, setupTestPageAndContextHooks, - describeFailsFirefox, } from './mocha-utils.js'; import {ConsoleMessage} from '../../lib/cjs/puppeteer/common/ConsoleMessage.js'; import { @@ -30,11 +29,12 @@ import { HTTPRequest, InterceptResolutionAction, } from '../../lib/cjs/puppeteer/common/HTTPRequest.js'; +import {it} from './mocha-utils.js'; describe('request interception', function () { setupTestBrowserHooks(); setupTestPageAndContextHooks(); - describeFailsFirefox('Page.setRequestInterception', function () { + describe('Page.setRequestInterception', function () { const expectedActions: ActionResult[] = ['abort', 'continue', 'respond']; while (expectedActions.length > 0) { const expectedAction = expectedActions.pop(); @@ -709,7 +709,7 @@ describe('request interception', function () { }); }); - describeFailsFirefox('Request.continue', function () { + describe('Request.continue', function () { it('should work', async () => { const {page, server} = getTestState(); @@ -805,7 +805,7 @@ describe('request interception', function () { }); }); - describeFailsFirefox('Request.respond', function () { + describe('Request.respond', function () { it('should work', async () => { const {page, server} = getTestState(); diff --git a/test/src/requestinterception.spec.ts b/test/src/requestinterception.spec.ts index bb7e0c03dd6..a1abc3ab5a9 100644 --- a/test/src/requestinterception.spec.ts +++ b/test/src/requestinterception.spec.ts @@ -22,15 +22,15 @@ import { getTestState, setupTestBrowserHooks, setupTestPageAndContextHooks, - describeFailsFirefox, } from './mocha-utils.js'; import {HTTPRequest} from '../../lib/cjs/puppeteer/common/HTTPRequest.js'; import {ConsoleMessage} from '../../lib/cjs/puppeteer/common/ConsoleMessage.js'; +import {it} from './mocha-utils.js'; describe('request interception', function () { setupTestBrowserHooks(); setupTestPageAndContextHooks(); - describeFailsFirefox('Page.setRequestInterception', function () { + describe('Page.setRequestInterception', function () { it('should intercept', async () => { const {page, server} = getTestState(); @@ -623,7 +623,7 @@ describe('request interception', function () { }); }); - describeFailsFirefox('Request.continue', function () { + describe('Request.continue', function () { it('should work', async () => { const {page, server} = getTestState(); @@ -739,7 +739,7 @@ describe('request interception', function () { }); }); - describeFailsFirefox('Request.respond', function () { + describe('Request.respond', function () { it('should work', async () => { const {page, server} = getTestState(); diff --git a/test/src/screenshot.spec.ts b/test/src/screenshot.spec.ts index 85ef96b203a..eefeda4bf61 100644 --- a/test/src/screenshot.spec.ts +++ b/test/src/screenshot.spec.ts @@ -19,17 +19,15 @@ import { getTestState, setupTestBrowserHooks, setupTestPageAndContextHooks, - itFailsFirefox, - itHeadfulOnly, - itChromeOnly, } from './mocha-utils.js'; +import {it} from './mocha-utils.js'; describe('Screenshots', function () { setupTestBrowserHooks(); setupTestPageAndContextHooks(); describe('Page.screenshot', function () { - itFailsFirefox('should work', async () => { + it('should work', async () => { const {page, server} = getTestState(); await page.setViewport({width: 500, height: 500}); @@ -37,7 +35,7 @@ describe('Screenshots', function () { const screenshot = await page.screenshot(); expect(screenshot).toBeGolden('screenshot-sanity.png'); }); - itFailsFirefox('should clip rect', async () => { + it('should clip rect', async () => { const {page, server} = getTestState(); await page.setViewport({width: 500, height: 500}); @@ -52,7 +50,7 @@ describe('Screenshots', function () { }); expect(screenshot).toBeGolden('screenshot-clip-rect.png'); }); - itFailsFirefox('should use scale for clip', async () => { + it('should use scale for clip', async () => { const {page, server} = getTestState(); await page.setViewport({width: 500, height: 500}); @@ -68,23 +66,20 @@ describe('Screenshots', function () { }); expect(screenshot).toBeGolden('screenshot-clip-rect-scale2.png'); }); - itFailsFirefox( - 'should get screenshot bigger than the viewport', - async () => { - const {page, server} = getTestState(); - await page.setViewport({width: 50, height: 50}); - await page.goto(server.PREFIX + '/grid.html'); - const screenshot = await page.screenshot({ - clip: { - x: 25, - y: 25, - width: 100, - height: 100, - }, - }); - expect(screenshot).toBeGolden('screenshot-offscreen-clip.png'); - } - ); + it('should get screenshot bigger than the viewport', async () => { + const {page, server} = getTestState(); + await page.setViewport({width: 50, height: 50}); + await page.goto(server.PREFIX + '/grid.html'); + const screenshot = await page.screenshot({ + clip: { + x: 25, + y: 25, + width: 100, + height: 100, + }, + }); + expect(screenshot).toBeGolden('screenshot-offscreen-clip.png'); + }); it('should run in parallel', async () => { const {page, server} = getTestState(); @@ -106,7 +101,7 @@ describe('Screenshots', function () { const screenshots = await Promise.all(promises); expect(screenshots[1]!).toBeGolden('grid-cell-1.png'); }); - itFailsFirefox('should take fullPage screenshots', async () => { + it('should take fullPage screenshots', async () => { const {page, server} = getTestState(); await page.setViewport({width: 500, height: 500}); @@ -147,7 +142,7 @@ describe('Screenshots', function () { }) ); }); - itFailsFirefox('should allow transparency', async () => { + it('should allow transparency', async () => { const {page, server} = getTestState(); await page.setViewport({width: 100, height: 100}); @@ -155,7 +150,7 @@ describe('Screenshots', function () { const screenshot = await page.screenshot({omitBackground: true}); expect(screenshot).toBeGolden('transparent.png'); }); - itFailsFirefox('should render white background on jpeg file', async () => { + it('should render white background on jpeg file', async () => { const {page, server} = getTestState(); await page.setViewport({width: 100, height: 100}); @@ -166,7 +161,7 @@ describe('Screenshots', function () { }); expect(screenshot).toBeGolden('white.jpg'); }); - itFailsFirefox('should work with webp', async () => { + it('should work with webp', async () => { const {page, server} = getTestState(); await page.setViewport({width: 100, height: 100}); @@ -190,7 +185,7 @@ describe('Screenshots', function () { }); expect(screenshot).toBeGolden('screenshot-clip-odd-size.png'); }); - itFailsFirefox('should return base64', async () => { + it('should return base64', async () => { const {page, server} = getTestState(); await page.setViewport({width: 500, height: 500}); @@ -206,7 +201,7 @@ describe('Screenshots', function () { 'screenshot-sanity.png' ); }); - itHeadfulOnly('should work in "fromSurface: false" mode', async () => { + it('should work in "fromSurface: false" mode', async () => { const {page, server} = getTestState(); await page.setViewport({width: 500, height: 500}); @@ -231,7 +226,7 @@ describe('Screenshots', function () { const screenshot = await elementHandle.screenshot(); expect(screenshot).toBeGolden('screenshot-element-bounding-box.png'); }); - itChromeOnly('should work with a null viewport', async () => { + it('should work with a null viewport', async () => { const {defaultBrowserOptions, puppeteer, server} = getTestState(); const browser = await puppeteer.launch({ @@ -271,14 +266,12 @@ describe('Screenshots', function () { const screenshot = await elementHandle.screenshot(); expect(screenshot).toBeGolden('screenshot-element-padding-border.png'); }); - itFailsFirefox( - 'should capture full element when larger than viewport', - async () => { - const {page} = getTestState(); + it('should capture full element when larger than viewport', async () => { + const {page} = getTestState(); - await page.setViewport({width: 500, height: 500}); + await page.setViewport({width: 500, height: 500}); - await page.setContent(` + await page.setContent(` something above
`); - const elementHandle = (await page.$('div.to-screenshot'))!; - const screenshot = await elementHandle.screenshot(); - expect(screenshot).toBeGolden( - 'screenshot-element-larger-than-viewport.png' - ); + const elementHandle = (await page.$('div.to-screenshot'))!; + const screenshot = await elementHandle.screenshot(); + expect(screenshot).toBeGolden( + 'screenshot-element-larger-than-viewport.png' + ); - expect( - await page.evaluate(() => { - return { - w: window.innerWidth, - h: window.innerHeight, - }; - }) - ).toEqual({w: 500, h: 500}); - } - ); + expect( + await page.evaluate(() => { + return { + w: window.innerWidth, + h: window.innerHeight, + }; + }) + ).toEqual({w: 500, h: 500}); + }); it('should scroll element into view', async () => { const {page} = getTestState(); @@ -336,7 +328,7 @@ describe('Screenshots', function () { 'screenshot-element-scrolled-into-view.png' ); }); - itFailsFirefox('should work with a rotated element', async () => { + it('should work with a rotated element', async () => { const {page} = getTestState(); await page.setViewport({width: 500, height: 500}); @@ -351,7 +343,7 @@ describe('Screenshots', function () { const screenshot = await elementHandle.screenshot(); expect(screenshot).toBeGolden('screenshot-element-rotate.png'); }); - itFailsFirefox('should fail to screenshot a detached element', async () => { + it('should fail to screenshot a detached element', async () => { const {page} = getTestState(); await page.setContent('

remove this

'); @@ -386,7 +378,7 @@ describe('Screenshots', function () { const screenshot = await elementHandle.screenshot(); expect(screenshot).toBeGolden('screenshot-element-fractional.png'); }); - itFailsFirefox('should work for an element with an offset', async () => { + it('should work for an element with an offset', async () => { const {page} = getTestState(); await page.setContent( diff --git a/test/src/target.spec.ts b/test/src/target.spec.ts index a67b31f0ee8..96fbc5fe359 100644 --- a/test/src/target.spec.ts +++ b/test/src/target.spec.ts @@ -20,11 +20,12 @@ import {Page} from '../../lib/cjs/puppeteer/common/Page.js'; import {Target} from '../../lib/cjs/puppeteer/common/Target.js'; import { getTestState, - itFailsFirefox, setupTestBrowserHooks, setupTestPageAndContextHooks, } from './mocha-utils.js'; import utils from './utils.js'; +import {it} from './mocha-utils.js'; + const {waitEvent} = utils; describe('Target', function () { @@ -79,7 +80,7 @@ describe('Target', function () { ).toBe('Hello world'); expect(await originalPage.$('body')).toBeTruthy(); }); - itFailsFirefox('should be able to use async waitForTarget', async () => { + it('should be able to use async waitForTarget', async () => { const {page, server, context} = getTestState(); const [otherPage] = await Promise.all([ @@ -101,89 +102,83 @@ describe('Target', function () { ); expect(page).not.toEqual(otherPage); }); - itFailsFirefox( - 'should report when a new page is created and closed', - async () => { - const {page, server, context} = getTestState(); + it('should report when a new page is created and closed', async () => { + const {page, server, context} = getTestState(); - const [otherPage] = await Promise.all([ - context - .waitForTarget(target => { - return target.url() === server.CROSS_PROCESS_PREFIX + '/empty.html'; - }) - .then(target => { - return target.page(); - }), - page.evaluate((url: string) => { - return window.open(url); - }, server.CROSS_PROCESS_PREFIX + '/empty.html'), - ]); - expect(otherPage!.url()).toContain(server.CROSS_PROCESS_PREFIX); - expect( - await otherPage!.evaluate(() => { - return ['Hello', 'world'].join(' '); + const [otherPage] = await Promise.all([ + context + .waitForTarget(target => { + return target.url() === server.CROSS_PROCESS_PREFIX + '/empty.html'; }) - ).toBe('Hello world'); - expect(await otherPage!.$('body')).toBeTruthy(); - - let allPages = await context.pages(); - expect(allPages).toContain(page); - expect(allPages).toContain(otherPage); - - const closePagePromise = new Promise(fulfill => { - return context.once('targetdestroyed', target => { - return fulfill(target.page()); - }); - }); - await otherPage!.close(); - expect(await closePagePromise).toBe(otherPage); - - allPages = (await Promise.all( - context.targets().map(target => { + .then(target => { return target.page(); - }) - )) as Page[]; - expect(allPages).toContain(page); - expect(allPages).not.toContain(otherPage); - } - ); - itFailsFirefox( - 'should report when a service worker is created and destroyed', - async () => { - const {page, server, context} = getTestState(); + }), + page.evaluate((url: string) => { + return window.open(url); + }, server.CROSS_PROCESS_PREFIX + '/empty.html'), + ]); + expect(otherPage!.url()).toContain(server.CROSS_PROCESS_PREFIX); + expect( + await otherPage!.evaluate(() => { + return ['Hello', 'world'].join(' '); + }) + ).toBe('Hello world'); + expect(await otherPage!.$('body')).toBeTruthy(); - await page.goto(server.EMPTY_PAGE); - const createdTarget = new Promise(fulfill => { - return context.once('targetcreated', target => { - return fulfill(target); - }); + let allPages = await context.pages(); + expect(allPages).toContain(page); + expect(allPages).toContain(otherPage); + + const closePagePromise = new Promise(fulfill => { + return context.once('targetdestroyed', target => { + return fulfill(target.page()); }); + }); + await otherPage!.close(); + expect(await closePagePromise).toBe(otherPage); - await page.goto(server.PREFIX + '/serviceworkers/empty/sw.html'); + allPages = (await Promise.all( + context.targets().map(target => { + return target.page(); + }) + )) as Page[]; + expect(allPages).toContain(page); + expect(allPages).not.toContain(otherPage); + }); + it('should report when a service worker is created and destroyed', async () => { + const {page, server, context} = getTestState(); - expect((await createdTarget).type()).toBe('service_worker'); - expect((await createdTarget).url()).toBe( - server.PREFIX + '/serviceworkers/empty/sw.js' - ); - - const destroyedTarget = new Promise(fulfill => { - return context.once('targetdestroyed', target => { - return fulfill(target); - }); + await page.goto(server.EMPTY_PAGE); + const createdTarget = new Promise(fulfill => { + return context.once('targetcreated', target => { + return fulfill(target); }); - await page.evaluate(() => { - return ( - globalThis as unknown as { - registrationPromise: Promise<{unregister: () => void}>; - } - ).registrationPromise.then((registration: any) => { - return registration.unregister(); - }); + }); + + await page.goto(server.PREFIX + '/serviceworkers/empty/sw.html'); + + expect((await createdTarget).type()).toBe('service_worker'); + expect((await createdTarget).url()).toBe( + server.PREFIX + '/serviceworkers/empty/sw.js' + ); + + const destroyedTarget = new Promise(fulfill => { + return context.once('targetdestroyed', target => { + return fulfill(target); }); - expect(await destroyedTarget).toBe(await createdTarget); - } - ); - itFailsFirefox('should create a worker from a service worker', async () => { + }); + await page.evaluate(() => { + return ( + globalThis as unknown as { + registrationPromise: Promise<{unregister: () => void}>; + } + ).registrationPromise.then((registration: any) => { + return registration.unregister(); + }); + }); + expect(await destroyedTarget).toBe(await createdTarget); + }); + it('should create a worker from a service worker', async () => { const {page, server, context} = getTestState(); await page.goto(server.PREFIX + '/serviceworkers/empty/sw.html'); @@ -198,7 +193,7 @@ describe('Target', function () { }) ).toBe('[object ServiceWorkerGlobalScope]'); }); - itFailsFirefox('should create a worker from a shared worker', async () => { + it('should create a worker from a shared worker', async () => { const {page, server, context} = getTestState(); await page.goto(server.EMPTY_PAGE); @@ -215,7 +210,7 @@ describe('Target', function () { }) ).toBe('[object SharedWorkerGlobalScope]'); }); - itFailsFirefox('should report when a target url changes', async () => { + it('should report when a target url changes', async () => { const {page, server, context} = getTestState(); await page.goto(server.EMPTY_PAGE); @@ -235,7 +230,7 @@ describe('Target', function () { await page.goto(server.EMPTY_PAGE); expect((await changedTarget).url()).toBe(server.EMPTY_PAGE); }); - itFailsFirefox('should not report uninitialized pages', async () => { + it('should not report uninitialized pages', async () => { const {context} = getTestState(); let targetChanged = false; @@ -268,37 +263,34 @@ describe('Target', function () { expect(targetChanged).toBe(false); context.removeListener('targetchanged', listener); }); - itFailsFirefox( - 'should not crash while redirecting if original request was missed', - async () => { - const {page, server, context} = getTestState(); + it('should not crash while redirecting if original request was missed', async () => { + const {page, server, context} = getTestState(); - let serverResponse!: ServerResponse; - server.setRoute('/one-style.css', (_req, res) => { - return (serverResponse = res); - }); - // Open a new page. Use window.open to connect to the page later. - await Promise.all([ - page.evaluate((url: string) => { - return window.open(url); - }, server.PREFIX + '/one-style.html'), - server.waitForRequest('/one-style.css'), - ]); - // Connect to the opened page. - const target = await context.waitForTarget(target => { - return target.url().includes('one-style.html'); - }); - const newPage = (await target.page())!; - // Issue a redirect. - serverResponse.writeHead(302, {location: '/injectedstyle.css'}); - serverResponse.end(); - // Wait for the new page to load. - await waitEvent(newPage, 'load'); - // Cleanup. - await newPage.close(); - } - ); - itFailsFirefox('should have an opener', async () => { + let serverResponse!: ServerResponse; + server.setRoute('/one-style.css', (_req, res) => { + return (serverResponse = res); + }); + // Open a new page. Use window.open to connect to the page later. + await Promise.all([ + page.evaluate((url: string) => { + return window.open(url); + }, server.PREFIX + '/one-style.html'), + server.waitForRequest('/one-style.css'), + ]); + // Connect to the opened page. + const target = await context.waitForTarget(target => { + return target.url().includes('one-style.html'); + }); + const newPage = (await target.page())!; + // Issue a redirect. + serverResponse.writeHead(302, {location: '/injectedstyle.css'}); + serverResponse.end(); + // Wait for the new page to load. + await waitEvent(newPage, 'load'); + // Cleanup. + await newPage.close(); + }); + it('should have an opener', async () => { const {page, server, context} = getTestState(); await page.goto(server.EMPTY_PAGE); @@ -318,7 +310,7 @@ describe('Target', function () { }); describe('Browser.waitForTarget', () => { - itFailsFirefox('should wait for a target', async () => { + it('should wait for a target', async () => { const {browser, puppeteer, server} = getTestState(); let resolved = false; diff --git a/test/src/touchscreen.spec.ts b/test/src/touchscreen.spec.ts index f32a2158ac2..12992674732 100644 --- a/test/src/touchscreen.spec.ts +++ b/test/src/touchscreen.spec.ts @@ -19,10 +19,10 @@ import { getTestState, setupTestBrowserHooks, setupTestPageAndContextHooks, - describeFailsFirefox, } from './mocha-utils.js'; +import {it} from './mocha-utils.js'; -describeFailsFirefox('Touchscreen', function () { +describe('Touchscreen', function () { setupTestBrowserHooks(); setupTestPageAndContextHooks(); diff --git a/test/src/tracing.spec.ts b/test/src/tracing.spec.ts index 2a7a4675cbc..27b1ae50837 100644 --- a/test/src/tracing.spec.ts +++ b/test/src/tracing.spec.ts @@ -17,11 +17,12 @@ import fs from 'fs'; import path from 'path'; import expect from 'expect'; -import {getTestState, describeChromeOnly} from './mocha-utils.js'; +import {getTestState} from './mocha-utils.js'; import {Browser} from '../../lib/cjs/puppeteer/common/Browser.js'; import {Page} from '../../lib/cjs/puppeteer/common/Page.js'; +import {it} from './mocha-utils.js'; -describeChromeOnly('Tracing', function () { +describe('Tracing', function () { let outputFile!: string; let browser!: Browser; let page!: Page; diff --git a/test/src/waittask.spec.ts b/test/src/waittask.spec.ts index 3ac049d418e..0988b5ac5b9 100644 --- a/test/src/waittask.spec.ts +++ b/test/src/waittask.spec.ts @@ -18,11 +18,11 @@ import expect from 'expect'; import {isErrorLike} from '../../lib/cjs/puppeteer/util/ErrorLike.js'; import { getTestState, - itFailsFirefox, setupTestBrowserHooks, setupTestPageAndContextHooks, } from './mocha-utils.js'; import {attachFrame, detachFrame} from './utils.js'; +import {it} from './mocha-utils.js'; describe('waittask specs', function () { setupTestBrowserHooks(); @@ -188,7 +188,7 @@ describe('waittask specs', function () { }); await watchdog; }); - itFailsFirefox('should work with strict CSP policy', async () => { + it('should work with strict CSP policy', async () => { const {page, server} = getTestState(); server.setCSP('/empty.html', 'script-src ' + server.PREFIX); @@ -421,7 +421,7 @@ describe('waittask specs', function () { await frame.waitForSelector('div'); }); - itFailsFirefox('should work with removed MutationObserver', async () => { + it('should work with removed MutationObserver', async () => { const {page} = getTestState(); await page.evaluate(() => { @@ -465,23 +465,20 @@ describe('waittask specs', function () { await watchdog; }); - itFailsFirefox( - 'Page.waitForSelector is shortcut for main frame', - async () => { - const {page, server} = getTestState(); + it('Page.waitForSelector is shortcut for main frame', async () => { + const {page, server} = getTestState(); - await page.goto(server.EMPTY_PAGE); - await attachFrame(page, 'frame1', server.EMPTY_PAGE); - const otherFrame = page.frames()[1]!; - const watchdog = page.waitForSelector('div'); - await otherFrame.evaluate(addElement, 'div'); - await page.evaluate(addElement, 'div'); - const eHandle = await watchdog; - expect(eHandle?.frame).toBe(page.mainFrame()); - } - ); + await page.goto(server.EMPTY_PAGE); + await attachFrame(page, 'frame1', server.EMPTY_PAGE); + const otherFrame = page.frames()[1]!; + const watchdog = page.waitForSelector('div'); + await otherFrame.evaluate(addElement, 'div'); + await page.evaluate(addElement, 'div'); + const eHandle = await watchdog; + expect(eHandle?.frame).toBe(page.mainFrame()); + }); - itFailsFirefox('should run in specified frame', async () => { + it('should run in specified frame', async () => { const {page, server} = getTestState(); await attachFrame(page, 'frame1', server.EMPTY_PAGE); @@ -495,7 +492,7 @@ describe('waittask specs', function () { expect(eHandle?.frame).toBe(frame2); }); - itFailsFirefox('should throw when frame is detached', async () => { + it('should throw when frame is detached', async () => { const {page, server} = getTestState(); await attachFrame(page, 'frame1', server.EMPTY_PAGE); @@ -738,7 +735,7 @@ describe('waittask specs', function () { 'waiting for selector `.//div` failed: timeout 10ms exceeded' ); }); - itFailsFirefox('should run in specified frame', async () => { + it('should run in specified frame', async () => { const {page, server} = getTestState(); await attachFrame(page, 'frame1', server.EMPTY_PAGE); @@ -751,7 +748,7 @@ describe('waittask specs', function () { const eHandle = await waitForXPathPromise; expect(eHandle?.frame).toBe(frame2); }); - itFailsFirefox('should throw when frame is detached', async () => { + it('should throw when frame is detached', async () => { const {page, server} = getTestState(); await attachFrame(page, 'frame1', server.EMPTY_PAGE); diff --git a/test/src/worker.spec.ts b/test/src/worker.spec.ts index c45febc3d4c..6ff8cfb1682 100644 --- a/test/src/worker.spec.ts +++ b/test/src/worker.spec.ts @@ -18,14 +18,14 @@ import expect from 'expect'; import {ConsoleMessage} from '../../lib/cjs/puppeteer/common/ConsoleMessage.js'; import {WebWorker} from '../../lib/cjs/puppeteer/common/WebWorker.js'; import { - describeFailsFirefox, getTestState, setupTestBrowserHooks, setupTestPageAndContextHooks, } from './mocha-utils.js'; import {waitEvent} from './utils.js'; +import {it} from './mocha-utils.js'; -describeFailsFirefox('Workers', function () { +describe('Workers', function () { setupTestBrowserHooks(); setupTestPageAndContextHooks(); it('Page.workers', async () => { diff --git a/test/tsconfig.json b/test/tsconfig.json index 962644f958d..6842bd8ed42 100644 --- a/test/tsconfig.json +++ b/test/tsconfig.json @@ -9,7 +9,8 @@ }, "include": ["src"], "references": [ - {"path": "../tsconfig.lib.json"}, - {"path": "../utils/testserver/tsconfig.json"} + {"path": "../src/tsconfig.cjs.json"}, + {"path": "../utils/testserver/tsconfig.json"}, + {"path": "../utils/mochaRunner/tsconfig.json"} ] } diff --git a/utils/mochaRunner/README.md b/utils/mochaRunner/README.md new file mode 100644 index 00000000000..efbafbc3b5e --- /dev/null +++ b/utils/mochaRunner/README.md @@ -0,0 +1,47 @@ +# Mocha Runner + +Mocha Runner is a test runner on top of mocha. It uses `/test/TestSuites.json` and `/test/TestExpectations.json` files to run mocha tests in multiple configurations and interpret results. + +## Running tests for Mocha Runner itself. + +``` +npm run build:dev && npx c8 node utils/mochaRunner/lib/test.js +``` + +## Running tests using Mocha Runner + +``` +npm run build:dev && npm run test +``` + +By default, the runner runs all test suites applicable to the current platform. +To pick a test suite, provide the `--test-suite` arguments. For example, + +``` +npm run build:dev && npm run test -- --test-suite chrome-headless +``` + +## TestSuites.json + +Define test suites via the `testSuites` attribute. `parameters` can be used in the `TestExpectations.json` to disable tests +based on parameters. The meaning for parameters is defined in `parameterDefinitons` which tell what env object corresponds +to the given parameter. + +## TestExpectations.json + +An expectation looks like this: + +``` + { + "testIdPattern": "[accessibility.spec]", + "platforms": ["darwin", "win32", "linux"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + } +``` + +`testIdPattern` defines a string that will be used to prefix-match tests. `platforms` defines the platforms the expectation is for (`or`-logic). +`parameters` defines the parameters that the test has to match (`and`-logic). `expectations` is the list of test results that are considered to be acceptable. + +Currently, expectations are updated manually. The test runner outputs the suggested changes to the expectation file if the test run does not match +expectations. diff --git a/utils/mochaRunner/src/main.ts b/utils/mochaRunner/src/main.ts new file mode 100644 index 00000000000..05f3168ce8f --- /dev/null +++ b/utils/mochaRunner/src/main.ts @@ -0,0 +1,204 @@ +import { + TestExpectation, + MochaResults, + zTestSuiteFile, + zPlatform, + TestSuite, + TestSuiteFile, + Platform, +} from './types.js'; + +import path from 'path'; +import fs from 'fs'; +import os from 'os'; +import {spawn} from 'node:child_process'; +import { + extendProcessEnv, + filterByPlatform, + prettyPrintJSON, + readJSON, + filterByParameters, + getExpectationUpdates, + getSkippedTests, +} from './utils.js'; + +function getApplicableTestSuites( + parsedSuitesFile: TestSuiteFile, + platform: Platform +): TestSuite[] { + const testSuiteArgIdx = process.argv.indexOf('--test-suite'); + let applicableSuites: TestSuite[] = []; + + if (testSuiteArgIdx === -1) { + applicableSuites = filterByPlatform(parsedSuitesFile.testSuites, platform); + } else { + const testSuiteId = process.argv[testSuiteArgIdx + 1]; + const testSuite = parsedSuitesFile.testSuites.find(suite => { + return suite.id === testSuiteId; + }); + + if (!testSuite) { + console.error(`Test suite ${testSuiteId} is not defined`); + process.exit(1); + } + + if (!testSuite.platforms.includes(platform)) { + console.warn( + `Test suite ${testSuiteId} is not enabled for your platform. Running it anyway.` + ); + } + + applicableSuites = [testSuite]; + } + + return applicableSuites; +} + +async function main() { + const platform = zPlatform.parse(os.platform()); + + const expectations = readJSON( + path.join(process.cwd(), 'test', 'TestExpectations.json') + ) as TestExpectation[]; + + const parsedSuitesFile = zTestSuiteFile.parse( + readJSON(path.join(process.cwd(), 'test', 'TestSuites.json')) + ); + + const applicableSuites = getApplicableTestSuites(parsedSuitesFile, platform); + + console.log('Planning to run the following test suites', applicableSuites); + + let fail = false; + const recommendations = []; + try { + for (const suite of applicableSuites) { + const parameters = suite.parameters; + + const applicableExpectations = filterByParameters( + filterByPlatform(expectations, platform), + parameters + ); + + const skippedTests = getSkippedTests(applicableExpectations); + + const env = extendProcessEnv([ + ...parameters.map(param => { + return parsedSuitesFile.parameterDefinitons[param]; + }), + { + PUPPETEER_SKIPPED_TEST_CONFIG: JSON.stringify( + skippedTests.map(ex => { + return ex.testIdPattern; + }) + ), + }, + ]); + + const tmpDir = fs.mkdtempSync( + path.join(os.tmpdir(), 'puppeteer-test-runner-') + ); + const tmpFilename = path.join(tmpDir, 'output.json'); + console.log('Running', JSON.stringify(parameters), tmpFilename); + const handle = spawn( + 'npx mocha', + ['--reporter=json', '--reporter-option', 'output=' + tmpFilename], + { + shell: true, + cwd: process.cwd(), + stdio: 'inherit', + env, + } + ); + await new Promise((resolve, reject) => { + handle.on('error', err => { + reject(err); + }); + handle.on('close', () => { + resolve(); + }); + }); + console.log('Finished', JSON.stringify(parameters)); + try { + const results = readJSON(tmpFilename) as MochaResults; + console.log('Results from mocha'); + console.log('Stats', JSON.stringify(results.stats)); + results.pending.length > 0 && console.log('# Pending tests'); + for (const test of results.pending) { + console.log(`\t? ${test.fullTitle} ${test.file}`); + } + results.failures.length > 0 && console.log('# Failed tests'); + for (const test of results.failures) { + console.log(`\tF ${test.fullTitle} ${test.file}`, test.err); + } + const recommendation = getExpectationUpdates( + results, + applicableExpectations, + { + platforms: [os.platform()], + parameters, + } + ); + if (recommendation.length > 0) { + fail = true; + recommendations.push(...recommendation); + } else { + console.log('Test run matches expecations'); + continue; + } + } catch (err) { + fail = true; + console.error(err); + } + } + } catch (err) { + fail = true; + console.error(err); + } finally { + const toAdd = recommendations.filter(item => { + return item.action === 'add'; + }); + if (toAdd.length) { + console.log( + 'Add the following to TestExpecations.json to ignore the error:' + ); + prettyPrintJSON( + toAdd.map(item => { + return item.expectation; + }) + ); + } + const toRemove = recommendations.filter(item => { + return item.action === 'remove'; + }); + if (toRemove.length) { + console.log( + 'Remove the following from the TestExpecations.json to ignore the error:' + ); + prettyPrintJSON( + toRemove.map(item => { + return item.expectation; + }) + ); + } + const toUpdate = recommendations.filter(item => { + return item.action === 'update'; + }); + if (toUpdate.length) { + console.log( + 'Update the following expectations in the TestExpecations.json to ignore the error:' + ); + prettyPrintJSON( + toUpdate.map(item => { + return item.expectation; + }) + ); + } + process.exit(fail ? 1 : 0); + } +} + +main().catch(error => { + console.error(error); + process.exit(1); +}); diff --git a/utils/mochaRunner/src/test.ts b/utils/mochaRunner/src/test.ts new file mode 100644 index 00000000000..2de6fbe4607 --- /dev/null +++ b/utils/mochaRunner/src/test.ts @@ -0,0 +1,46 @@ +import assert from 'assert/strict'; +import test from 'node:test'; +import {filterByParameters, getTestResultForFailure} from './utils.js'; +import {TestExpectation} from './types.js'; +import {getFilename, extendProcessEnv} from './utils.js'; + +test('extendProcessEnv', () => { + const env = extendProcessEnv([{TEST: 'TEST'}, {TEST2: 'TEST2'}]); + assert.equal(env['TEST'], 'TEST'); + assert.equal(env['TEST2'], 'TEST2'); +}); + +test('getFilename', () => { + assert.equal(getFilename('/etc/test.ts'), 'test'); + assert.equal(getFilename('/etc/test.js'), 'test'); +}); + +test('getTestResultForFailure', () => { + assert.equal( + getTestResultForFailure({err: {code: 'ERR_MOCHA_TIMEOUT'}}), + 'TIMEOUT' + ); + assert.equal(getTestResultForFailure({err: {code: 'ERROR'}}), 'FAIL'); +}); + +test('filterByParameters', () => { + const expectations: TestExpectation[] = [ + { + testIdPattern: + '[oopif.spec] OOPIF "after all" hook for "should keep track of a frames OOP state"', + platforms: ['darwin'], + parameters: ['firefox', 'headless'], + expectations: ['FAIL'], + }, + ]; + assert.equal( + filterByParameters(expectations, ['firefox', 'headless']).length, + 1 + ); + assert.equal(filterByParameters(expectations, ['firefox']).length, 0); + assert.equal( + filterByParameters(expectations, ['firefox', 'headless', 'other']).length, + 1 + ); + assert.equal(filterByParameters(expectations, ['other']).length, 0); +}); diff --git a/utils/mochaRunner/src/types.ts b/utils/mochaRunner/src/types.ts new file mode 100644 index 00000000000..7fe7b711686 --- /dev/null +++ b/utils/mochaRunner/src/types.ts @@ -0,0 +1,42 @@ +import {z} from 'zod'; + +export const zPlatform = z.enum(['win32', 'linux', 'darwin']); + +export type Platform = z.infer; + +export const zTestSuite = z.object({ + id: z.string(), + platforms: z.array(zPlatform), + parameters: z.array(z.string()), +}); + +export type TestSuite = z.infer; + +export const zTestSuiteFile = z.object({ + testSuites: z.array(zTestSuite), + parameterDefinitons: z.record(z.any()), +}); + +export type TestSuiteFile = z.infer; + +export type TestResult = 'PASS' | 'FAIL' | 'TIMEOUT' | 'SKIP'; + +export type TestExpectation = { + testIdPattern: string; + platforms: NodeJS.Platform[]; + parameters: string[]; + expectations: TestResult[]; +}; + +export type MochaTestResult = { + fullTitle: string; + file: string; + err?: {code: string}; +}; + +export type MochaResults = { + stats: unknown; + pending: MochaTestResult[]; + passes: MochaTestResult[]; + failures: MochaTestResult[]; +}; diff --git a/utils/mochaRunner/src/utils.ts b/utils/mochaRunner/src/utils.ts new file mode 100644 index 00000000000..d6deb0efa7a --- /dev/null +++ b/utils/mochaRunner/src/utils.ts @@ -0,0 +1,153 @@ +import { + MochaTestResult, + TestExpectation, + MochaResults, + TestResult, +} from './types.js'; +import path from 'path'; +import fs from 'fs'; + +export function extendProcessEnv(envs: object[]): NodeJS.ProcessEnv { + return envs.reduce( + (acc: object, item: object) => { + Object.assign(acc, item); + return acc; + }, + { + ...process.env, + } + ) as NodeJS.ProcessEnv; +} + +export function getFilename(file: string): string { + return path.basename(file).replace(path.extname(file), ''); +} + +export function readJSON(path: string): unknown { + return JSON.parse(fs.readFileSync(path, 'utf-8')); +} + +export function filterByPlatform( + items: T[], + platform: NodeJS.Platform +): T[] { + return items.filter(item => { + return item.platforms.includes(platform); + }); +} + +export function prettyPrintJSON(json: unknown): void { + console.log(JSON.stringify(json, null, 2)); +} + +export function filterByParameters( + expecations: TestExpectation[], + parameters: string[] +): TestExpectation[] { + const querySet = new Set(parameters); + return expecations.filter(ex => { + return ex.parameters.every(param => { + return querySet.has(param); + }); + }); +} + +/** + * The last expectation that matches the startsWith filter wins. + */ +export function findEffectiveExpecationForTest( + expectations: TestExpectation[], + result: MochaTestResult +): TestExpectation | undefined { + return expectations + .filter(expecation => { + if ( + getTestId(result.file, result.fullTitle).startsWith( + expecation.testIdPattern + ) + ) { + return true; + } + return false; + }) + .pop(); +} + +type RecommendedExpecation = { + expectation: TestExpectation; + test: MochaTestResult; + action: 'remove' | 'add' | 'update'; +}; + +export function getExpectationUpdates( + results: MochaResults, + expecations: TestExpectation[], + context: { + platforms: NodeJS.Platform[]; + parameters: string[]; + } +): RecommendedExpecation[] { + const output: RecommendedExpecation[] = []; + + for (const pass of results.passes) { + const expectation = findEffectiveExpecationForTest(expecations, pass); + if (expectation && !expectation.expectations.includes('PASS')) { + output.push({ + expectation, + test: pass, + action: 'remove', + }); + } + } + + for (const failure of results.failures) { + const expectation = findEffectiveExpecationForTest(expecations, failure); + if (expectation) { + if ( + !expectation.expectations.includes(getTestResultForFailure(failure)) + ) { + output.push({ + expectation: { + ...expectation, + expectations: [ + ...expectation.expectations, + getTestResultForFailure(failure), + ], + }, + test: failure, + action: 'update', + }); + } + } else { + output.push({ + expectation: { + testIdPattern: getTestId(failure.file, failure.fullTitle), + platforms: context.platforms, + parameters: context.parameters, + expectations: [getTestResultForFailure(failure)], + }, + test: failure, + action: 'add', + }); + } + } + return output; +} + +export function getTestResultForFailure( + test: Pick +): TestResult { + return test.err?.code === 'ERR_MOCHA_TIMEOUT' ? 'TIMEOUT' : 'FAIL'; +} + +export function getSkippedTests( + expectations: TestExpectation[] +): TestExpectation[] { + return expectations.filter(ex => { + return ex.expectations.includes('SKIP'); + }); +} + +export function getTestId(file: string, fullTitle: string): string { + return `[${getFilename(file)}] ${fullTitle}`; +} diff --git a/utils/mochaRunner/tsconfig.json b/utils/mochaRunner/tsconfig.json new file mode 100644 index 00000000000..c2576c2564f --- /dev/null +++ b/utils/mochaRunner/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "allowJs": true, + "composite": true, + "module": "CommonJS", + "outDir": "lib", + "rootDir": "src" + }, + "include": ["src"] +}