fix: disable GFX sanity window for Firefox and enable WebDriver BiDi CI jobs for Windows (#11578)

This commit is contained in:
Henrik Skupin 2023-12-28 13:20:22 +01:00 committed by GitHub
parent 107b8337e5
commit e41a2656d9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 141 additions and 64 deletions

View File

@ -231,6 +231,7 @@ jobs:
os:
- ubuntu-latest
- macos-latest
- windows-latest
suite:
- firefox-bidi
- firefox-headful
@ -249,6 +250,10 @@ jobs:
# The first time the browsers launches it crashes.
- os: macos-latest
suite: firefox-bidi
- os: windows-latest
suite: firefox-headful
- os: windows-latest
suite: firefox-headless
steps:
- name: Checkout
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1

View File

@ -222,20 +222,29 @@ function defaultProfilePreferences(
// Allow the application to have focus even it runs in the background
'focusmanager.testmode': true,
// Disable useragent updates
'general.useragent.updates.enabled': false,
// Always use network provider for geolocation tests so we bypass the
// macOS dialog raised by the corelocation provider
'geo.provider.testing': true,
// Do not scan Wifi
'geo.wifi.scan': false,
// No hang monitor
'hangmonitor.timeout': 0,
// Show chrome errors and warnings in the error console
'javascript.options.showInConsole': true,
// Disable download and usage of OpenH264: and Widevine plugins
'media.gmp-manager.updateEnabled': false,
// Disable the GFX sanity window
'media.sanity-test.disabled': true,
// Prevent various error message on the console
// jest-puppeteer asserts that no error message is emitted by the console
'network.cookie.cookieBehavior': 0,
@ -267,9 +276,11 @@ function defaultProfilePreferences(
// Don't do network connections for mitm priming
'security.certerrors.mitm.priming.enabled': false,
// Local documents have access to all other local documents,
// including directory listings
'security.fileuri.strict_origin_policy': false,
// Do not wait for the notification button security delay
'security.notification_enable_delay': 0,
@ -279,6 +290,7 @@ function defaultProfilePreferences(
// Do not automatically fill sign-in forms with known usernames and
// passwords
'signon.autofillForms': false,
// Disable password capture, so that tests that include forms are not
// influenced by the presence of the persistent doorhanger notification
'signon.rememberSignons': false,

View File

@ -109,10 +109,16 @@
},
{
"testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.launch *",
"platforms": ["darwin", "linux", "win32"],
"platforms": ["darwin", "linux"],
"parameters": ["webDriverBiDi"],
"expectations": ["PASS"]
},
{
"testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.launch *",
"platforms": ["win32"],
"parameters": ["webDriverBiDi"],
"expectations": ["SKIP"]
},
{
"testIdPattern": "[locator.spec] *",
"platforms": ["darwin", "linux", "win32"],
@ -151,20 +157,26 @@
},
{
"testIdPattern": "[network.spec] network Page.authenticate *",
"platforms": ["darwin", "linux", "win32"],
"platforms": ["darwin", "linux"],
"parameters": ["webDriverBiDi"],
"expectations": ["FAIL", "TIMEOUT"]
},
{
"testIdPattern": "[network.spec] network Page.setBypassServiceWorker *",
"platforms": ["darwin", "linux", "win32"],
"testIdPattern": "[network.spec] network Page.authenticate *",
"platforms": ["win32"],
"parameters": ["webDriverBiDi"],
"expectations": ["FAIL", "TIMEOUT"]
"expectations": ["SKIP"]
},
{
"testIdPattern": "[network.spec] network Page.setBypassServiceWorker *",
"platforms": ["darwin", "linux", "win32"],
"parameters": ["firefox"],
"expectations": ["SKIP"]
},
{
"testIdPattern": "[network.spec] network Page.setBypassServiceWorker *",
"platforms": ["darwin", "linux", "win32"],
"parameters": ["webDriverBiDi"],
"expectations": ["FAIL", "TIMEOUT"]
},
{
@ -315,7 +327,7 @@
"testIdPattern": "[screenshot.spec] Screenshots Cdp *",
"platforms": ["darwin", "linux", "win32"],
"parameters": ["webDriverBiDi"],
"expectations": ["FAIL"]
"expectations": ["SKIP"]
},
{
"testIdPattern": "[stacktrace.spec] Stack trace *",
@ -507,7 +519,7 @@
"testIdPattern": "[elementhandle.spec] ElementHandle specs ElementHandle.clickablePoint should not work if the click box is not visible due to the iframe",
"platforms": ["darwin", "linux", "win32"],
"parameters": ["webDriverBiDi"],
"expectations": ["TIMEOUT"]
"expectations": ["SKIP"]
},
{
"testIdPattern": "[emulation.spec] *",
@ -635,6 +647,12 @@
"parameters": ["webDriverBiDi"],
"expectations": ["TIMEOUT"]
},
{
"testIdPattern": "[launcher.spec] Launcher specs Puppeteer Browser.disconnect *",
"platforms": ["win32"],
"parameters": ["firefox", "webDriverBiDi"],
"expectations": ["SKIP"]
},
{
"testIdPattern": "[launcher.spec] Launcher specs Puppeteer Browser.disconnect should reject navigation when browser closes",
"platforms": ["darwin", "linux", "win32"],
@ -683,12 +701,6 @@
"parameters": ["chrome"],
"expectations": ["SKIP"]
},
{
"testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.launch should be able to launch Firefox",
"platforms": ["darwin", "linux", "win32"],
"parameters": ["webDriverBiDi"],
"expectations": ["FAIL"]
},
{
"testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.launch should have custom URL when launching browser",
"platforms": ["darwin", "linux", "win32"],
@ -713,6 +725,12 @@
"parameters": ["webDriverBiDi"],
"expectations": ["FAIL"]
},
{
"testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.launch should work with no default arguments",
"platforms": ["darwin", "linux", "win32"],
"parameters": ["firefox"],
"expectations": ["SKIP"]
},
{
"testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.launch tmp profile should be cleaned up",
"platforms": ["darwin", "linux", "win32"],
@ -785,6 +803,12 @@
"parameters": ["webDriverBiDi"],
"expectations": ["PASS"]
},
{
"testIdPattern": "[network.spec] network Page.setBypassServiceWorker *",
"platforms": ["win32"],
"parameters": ["firefox", "webDriverBiDi"],
"expectations": ["SKIP"]
},
{
"testIdPattern": "[network.spec] network raw network headers Cross-origin set-cookie",
"platforms": ["darwin", "linux", "win32"],
@ -1107,13 +1131,13 @@
"testIdPattern": "[queryhandler.spec] Query handler tests P selectors should work ARIA selectors with role",
"platforms": ["darwin", "linux", "win32"],
"parameters": ["webDriverBiDi"],
"expectations": ["FAIL"]
"expectations": ["SKIP"]
},
{
"testIdPattern": "[queryhandler.spec] Query handler tests P selectors should work for ARIA selectors in multiple isolated worlds",
"platforms": ["darwin", "linux", "win32"],
"parameters": ["webDriverBiDi"],
"expectations": ["FAIL", "TIMEOUT"]
"expectations": ["SKIP"]
},
{
"testIdPattern": "[queryhandler.spec] Query handler tests P selectors should work with :hover",
@ -2087,6 +2111,12 @@
"parameters": ["cdp", "firefox"],
"expectations": ["SKIP"]
},
{
"testIdPattern": "[launcher.spec] Launcher specs Puppeteer Browser.disconnect should reject navigation when browser closes",
"platforms": ["darwin", "linux", "win32"],
"parameters": ["cdp", "firefox"],
"expectations": ["FAIL", "PASS"]
},
{
"testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.connect should be able to close remote browser",
"platforms": ["darwin", "linux", "win32"],
@ -2165,12 +2195,24 @@
"parameters": ["chrome", "webDriverBiDi"],
"expectations": ["PASS"]
},
{
"testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.launch can launch and close the browser",
"platforms": ["win32"],
"parameters": ["firefox", "webDriverBiDi"],
"expectations": ["SKIP"]
},
{
"testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.launch should be able to launch Chrome",
"platforms": ["darwin", "linux", "win32"],
"parameters": ["chrome", "webDriverBiDi"],
"expectations": ["FAIL"]
},
{
"testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.launch should be able to launch Firefox",
"platforms": ["darwin", "linux", "win32"],
"parameters": ["firefox", "webDriverBiDi"],
"expectations": ["FAIL"]
},
{
"testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.launch should be able to launch Firefox",
"platforms": ["darwin", "linux", "win32"],
@ -2195,12 +2237,6 @@
"parameters": ["cdp", "firefox"],
"expectations": ["FAIL", "PASS"]
},
{
"testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.launch should work with no default arguments",
"platforms": ["darwin", "linux", "win32"],
"parameters": ["firefox", "headless"],
"expectations": ["SKIP"]
},
{
"testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.launch tmp profile should be cleaned up",
"platforms": ["darwin", "linux", "win32"],
@ -2265,7 +2301,7 @@
"testIdPattern": "[navigation.spec] navigation Frame.goto should navigate subframes",
"platforms": ["darwin", "linux", "win32"],
"parameters": ["firefox", "webDriverBiDi"],
"expectations": ["FAIL"]
"expectations": ["FAIL", "PASS"]
},
{
"testIdPattern": "[navigation.spec] navigation Frame.goto should reject when frame detaches",
@ -2301,7 +2337,7 @@
"testIdPattern": "[navigation.spec] navigation Frame.waitForNavigation should work",
"platforms": ["darwin", "linux", "win32"],
"parameters": ["firefox", "webDriverBiDi"],
"expectations": ["FAIL", "TIMEOUT"]
"expectations": ["FAIL", "PASS"]
},
{
"testIdPattern": "[navigation.spec] navigation Page.goBack should work with HistoryAPI",
@ -2637,7 +2673,7 @@
"testIdPattern": "[network.spec] network Request.frame should work for subframe navigation request",
"platforms": ["darwin", "linux", "win32"],
"parameters": ["firefox", "webDriverBiDi"],
"expectations": ["FAIL"]
"expectations": ["FAIL", "PASS"]
},
{
"testIdPattern": "[network.spec] network Request.headers should define Firefox as user agent header",
@ -2705,6 +2741,12 @@
"parameters": ["chrome", "webDriverBiDi"],
"expectations": ["FAIL"]
},
{
"testIdPattern": "[network.spec] network Response.fromServiceWorker Response.fromServiceWorker",
"platforms": ["win32"],
"parameters": ["firefox", "webDriverBiDi"],
"expectations": ["SKIP"]
},
{
"testIdPattern": "[network.spec] network Response.fromServiceWorker Response.fromServiceWorker",
"platforms": ["darwin", "linux", "win32"],
@ -2741,6 +2783,12 @@
"parameters": ["cdp", "firefox"],
"expectations": ["SKIP"]
},
{
"testIdPattern": "[network.spec] network Response.text should wait until response completes",
"platforms": ["win32"],
"parameters": ["firefox", "webDriverBiDi"],
"expectations": ["SKIP"]
},
{
"testIdPattern": "[network.spec] network Response.text should work",
"platforms": ["darwin", "linux", "win32"],
@ -3359,6 +3407,18 @@
"parameters": ["cdp", "chrome"],
"expectations": ["FAIL", "PASS"]
},
{
"testIdPattern": "[screenshot.spec] Screenshots Cdp should use scale for clip",
"platforms": ["darwin", "linux", "win32"],
"parameters": ["cdp", "firefox"],
"expectations": ["FAIL", "PASS"]
},
{
"testIdPattern": "[screenshot.spec] Screenshots ElementHandle.screenshot should capture full element when larger than viewport",
"platforms": ["win32"],
"parameters": ["cdp", "firefox"],
"expectations": ["FAIL"]
},
{
"testIdPattern": "[screenshot.spec] Screenshots ElementHandle.screenshot should work for an element with an offset",
"platforms": ["darwin", "linux", "win32"],
@ -3371,17 +3431,35 @@
"parameters": ["firefox", "webDriverBiDi"],
"expectations": ["FAIL"]
},
{
"testIdPattern": "[screenshot.spec] Screenshots Page.screenshot should clip clip bigger than the viewport without \"captureBeyondViewport\"",
"platforms": ["darwin", "linux", "win32"],
"parameters": ["cdp", "firefox"],
"expectations": ["FAIL", "PASS"]
},
{
"testIdPattern": "[screenshot.spec] Screenshots Page.screenshot should clip rect",
"platforms": ["darwin", "linux", "win32"],
"parameters": ["firefox", "webDriverBiDi"],
"expectations": ["FAIL"]
},
{
"testIdPattern": "[screenshot.spec] Screenshots Page.screenshot should clip rect",
"platforms": ["darwin", "linux", "win32"],
"parameters": ["cdp", "firefox"],
"expectations": ["FAIL", "PASS"]
},
{
"testIdPattern": "[screenshot.spec] Screenshots Page.screenshot should return base64",
"platforms": ["darwin", "linux", "win32"],
"parameters": ["firefox", "webDriverBiDi"],
"expectations": ["FAIL"]
},
{
"testIdPattern": "[screenshot.spec] Screenshots Page.screenshot should return base64",
"platforms": ["darwin", "linux", "win32"],
"parameters": ["firefox", "webDriverBiDi"],
"expectations": ["FAIL"]
"parameters": ["cdp", "firefox"],
"expectations": ["FAIL", "PASS"]
},
{
"testIdPattern": "[screenshot.spec] Screenshots Page.screenshot should take fullPage screenshots",
@ -3395,12 +3473,24 @@
"parameters": ["firefox", "webDriverBiDi"],
"expectations": ["FAIL"]
},
{
"testIdPattern": "[screenshot.spec] Screenshots Page.screenshot should take fullPage screenshots without captureBeyondViewport",
"platforms": ["darwin", "linux", "win32"],
"parameters": ["cdp", "firefox"],
"expectations": ["FAIL", "PASS"]
},
{
"testIdPattern": "[screenshot.spec] Screenshots Page.screenshot should work",
"platforms": ["darwin", "linux", "win32"],
"parameters": ["firefox", "webDriverBiDi"],
"expectations": ["FAIL"]
},
{
"testIdPattern": "[screenshot.spec] Screenshots Page.screenshot should work",
"platforms": ["darwin", "linux", "win32"],
"parameters": ["cdp", "firefox"],
"expectations": ["FAIL", "PASS"]
},
{
"testIdPattern": "[screenshot.spec] Screenshots Page.screenshot should work with odd clip size on Retina displays",
"platforms": ["darwin", "linux", "win32"],
@ -3579,7 +3669,7 @@
"testIdPattern": "[waittask.spec] waittask specs Frame.waitForSelector should survive cross-process navigation",
"platforms": ["darwin", "linux", "win32"],
"parameters": ["firefox", "webDriverBiDi"],
"expectations": ["PASS"]
"expectations": ["FAIL", "PASS", "TIMEOUT"]
},
{
"testIdPattern": "[waittask.spec] waittask specs Frame.waitForSelector should survive cross-process navigation",
@ -3654,10 +3744,10 @@
"expectations": ["FAIL"]
},
{
"testIdPattern": "[launcher.spec] Launcher specs Puppeteer Browser.disconnect should reject navigation when browser closes",
"platforms": ["darwin", "linux", "win32"],
"parameters": ["cdp", "firefox", "headful"],
"expectations": ["FAIL", "PASS"]
"testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.launch userDataDir option restores preferences",
"platforms": ["win32"],
"parameters": ["firefox", "headless", "webDriverBiDi"],
"expectations": ["SKIP"]
},
{
"testIdPattern": "[navigation.spec] navigation \"after all\" hook in \"navigation\"",
@ -3708,9 +3798,9 @@
"expectations": ["FAIL", "PASS"]
},
{
"testIdPattern": "[screenshot.spec] Screenshots Cdp should use scale for clip",
"platforms": ["darwin", "linux", "win32"],
"parameters": ["cdp", "firefox", "headless"],
"testIdPattern": "[screenshot.spec] Screenshots ElementHandle.screenshot should capture full element when larger than viewport",
"platforms": ["win32"],
"parameters": ["firefox", "headless", "webDriverBiDi"],
"expectations": ["FAIL"]
},
{
@ -3737,24 +3827,6 @@
"parameters": ["cdp", "firefox", "headless"],
"expectations": ["FAIL"]
},
{
"testIdPattern": "[screenshot.spec] Screenshots Page.screenshot should clip clip bigger than the viewport without \"captureBeyondViewport\"",
"platforms": ["darwin", "linux", "win32"],
"parameters": ["cdp", "firefox", "headless"],
"expectations": ["FAIL"]
},
{
"testIdPattern": "[screenshot.spec] Screenshots Page.screenshot should clip rect",
"platforms": ["darwin", "linux", "win32"],
"parameters": ["cdp", "firefox", "headless"],
"expectations": ["FAIL"]
},
{
"testIdPattern": "[screenshot.spec] Screenshots Page.screenshot should return base64",
"platforms": ["darwin", "linux", "win32"],
"parameters": ["cdp", "firefox", "headless"],
"expectations": ["FAIL"]
},
{
"testIdPattern": "[screenshot.spec] Screenshots Page.screenshot should take fullPage screenshots",
"platforms": ["darwin", "linux", "win32"],
@ -3767,18 +3839,6 @@
"parameters": ["cdp", "firefox", "headless"],
"expectations": ["FAIL"]
},
{
"testIdPattern": "[screenshot.spec] Screenshots Page.screenshot should take fullPage screenshots without captureBeyondViewport",
"platforms": ["darwin", "linux", "win32"],
"parameters": ["cdp", "firefox", "headless"],
"expectations": ["FAIL"]
},
{
"testIdPattern": "[screenshot.spec] Screenshots Page.screenshot should work",
"platforms": ["darwin", "linux", "win32"],
"parameters": ["cdp", "firefox", "headless"],
"expectations": ["FAIL"]
},
{
"testIdPattern": "[worker.spec] Workers Page.workers",
"platforms": ["darwin", "linux", "win32"],