From 5721e66cb9473896b8f0fbeaf782553ec9747b40 Mon Sep 17 00:00:00 2001 From: Alex Rudenko Date: Thu, 15 Feb 2024 10:57:06 +0100 Subject: [PATCH] build: auto-remove duplicate expectations (#11922) --- test/TestExpectations.json | 144 ------------------------------- tools/sort-test-expectations.mjs | 61 ++++++++++++- 2 files changed, 60 insertions(+), 145 deletions(-) diff --git a/test/TestExpectations.json b/test/TestExpectations.json index c9670a974eb..4d525ef5928 100644 --- a/test/TestExpectations.json +++ b/test/TestExpectations.json @@ -173,12 +173,6 @@ "parameters": ["webDriverBiDi"], "expectations": ["PASS"] }, - { - "testIdPattern": "[navigation.spec] navigation Page.goBack *", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["webDriverBiDi"], - "expectations": ["PASS"] - }, { "testIdPattern": "[network.spec] network *", "platforms": ["darwin", "linux", "win32"], @@ -587,12 +581,6 @@ "parameters": ["firefox"], "expectations": ["FAIL", "PASS"] }, - { - "testIdPattern": "[elementhandle.spec] ElementHandle specs ElementHandle.click should return Point data", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["webDriverBiDi"], - "expectations": ["PASS"] - }, { "testIdPattern": "[elementhandle.spec] ElementHandle specs ElementHandle.clickablePoint should not work if the click box is not visible due to the iframe", "platforms": ["darwin", "linux", "win32"], @@ -605,18 +593,6 @@ "parameters": ["chrome", "webDriverBiDi"], "expectations": ["PASS"] }, - { - "testIdPattern": "[emulation.spec] Emulation Page.viewport should get the proper viewport size", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["webDriverBiDi"], - "expectations": ["PASS"] - }, - { - "testIdPattern": "[emulation.spec] Emulation Page.viewport should support mobile emulation", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["webDriverBiDi"], - "expectations": ["PASS"] - }, { "testIdPattern": "[evaluation.spec] Evaluation specs Page.evaluate should replace symbols with undefined", "platforms": ["darwin", "linux", "win32"], @@ -720,12 +696,6 @@ "parameters": ["webDriverBiDi"], "expectations": ["FAIL"] }, - { - "testIdPattern": "[jshandle.spec] JSHandle Page.evaluateHandle should use the same JS wrappers", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["webDriverBiDi"], - "expectations": ["PASS"] - }, { "testIdPattern": "[keyboard.spec] Keyboard should send a character with sendCharacter in iframe", "platforms": ["darwin", "linux", "win32"], @@ -1010,12 +980,6 @@ "parameters": ["webDriverBiDi"], "expectations": ["SKIP"] }, - { - "testIdPattern": "[page.spec] Page Page.Events.Console should work for different console API calls with logging functions", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["webDriverBiDi"], - "expectations": ["PASS"] - }, { "testIdPattern": "[page.spec] Page Page.Events.error should throw when page crashes", "platforms": ["darwin", "linux", "win32"], @@ -1083,12 +1047,6 @@ "parameters": ["firefox", "webDriverBiDi"], "expectations": ["FAIL"] }, - { - "testIdPattern": "[page.spec] Page Page.waitForNetworkIdle should work with aborted requests", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["webDriverBiDi"], - "expectations": ["PASS"] - }, { "testIdPattern": "[pdf.spec] Page.pdf *", "platforms": ["darwin", "linux", "win32"], @@ -1149,18 +1107,6 @@ "parameters": ["webDriverBiDi"], "expectations": ["SKIP"] }, - { - "testIdPattern": "[queryhandler.spec] Query handler tests P selectors should work with :hover", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["webDriverBiDi"], - "expectations": ["PASS"] - }, - { - "testIdPattern": "[queryhandler.spec] Query handler tests Text selectors in Page should clear caches", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["webDriverBiDi"], - "expectations": ["PASS"] - }, { "testIdPattern": "[queryObjects.spec] *", "platforms": ["darwin", "linux", "win32"], @@ -1325,12 +1271,6 @@ "parameters": ["firefox", "webDriverBiDi"], "expectations": ["SKIP"] }, - { - "testIdPattern": "[accessibility.spec] Accessibility filtering children of leaf nodes rich text editable fields should have children", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["chrome", "webDriverBiDi"], - "expectations": ["PASS"] - }, { "testIdPattern": "[accessibility.spec] Accessibility get snapshots while the tree is re-calculated", "platforms": ["darwin", "linux", "win32"], @@ -1403,12 +1343,6 @@ "parameters": ["chrome", "webDriverBiDi"], "expectations": ["PASS"] }, - { - "testIdPattern": "[browser.spec] Browser specs Browser.isConnected should set the browser connected state", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["chrome", "webDriverBiDi"], - "expectations": ["PASS"] - }, { "testIdPattern": "[browser.spec] Browser specs Browser.process should not return child_process for remote browser", "platforms": ["darwin", "linux", "win32"], @@ -1487,18 +1421,6 @@ "parameters": ["cdp", "firefox"], "expectations": ["FAIL"] }, - { - "testIdPattern": "[browsercontext.spec] BrowserContext should close all belonging targets once closing context", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["chrome", "webDriverBiDi"], - "expectations": ["PASS"] - }, - { - "testIdPattern": "[browsercontext.spec] BrowserContext should create new incognito context", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["chrome", "webDriverBiDi"], - "expectations": ["PASS"] - }, { "testIdPattern": "[browsercontext.spec] BrowserContext should create new incognito context", "platforms": ["darwin", "linux", "win32"], @@ -1524,24 +1446,12 @@ "parameters": ["firefox", "webDriverBiDi"], "expectations": ["PASS"] }, - { - "testIdPattern": "[browsercontext.spec] BrowserContext should timeout waiting for a non-existent target", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["chrome", "webDriverBiDi"], - "expectations": ["PASS"] - }, { "testIdPattern": "[browsercontext.spec] BrowserContext should wait for a target", "platforms": ["darwin", "linux", "win32"], "parameters": ["cdp", "firefox"], "expectations": ["SKIP"] }, - { - "testIdPattern": "[browsercontext.spec] BrowserContext should wait for a target", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["chrome", "webDriverBiDi"], - "expectations": ["PASS"] - }, { "testIdPattern": "[browsercontext.spec] BrowserContext should work across sessions", "platforms": ["darwin", "linux", "win32"], @@ -1842,18 +1752,6 @@ "parameters": ["cdp", "firefox"], "expectations": ["FAIL"] }, - { - "testIdPattern": "[coverage.spec] Coverage specs CSSCoverage should work with complicated usecases", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["chrome", "webDriverBiDi"], - "expectations": ["PASS"] - }, - { - "testIdPattern": "[coverage.spec] Coverage specs JSCoverage should not ignore eval() scripts if reportAnonymousScripts is true", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["chrome", "webDriverBiDi"], - "expectations": ["PASS"] - }, { "testIdPattern": "[debugInfo.spec] DebugInfo Browser.debugInfo should work", "platforms": ["darwin", "linux", "win32"], @@ -2112,24 +2010,12 @@ "parameters": ["firefox", "webDriverBiDi"], "expectations": ["FAIL"] }, - { - "testIdPattern": "[emulation.spec] Emulation Page.viewport should detect touch when applying viewport with touches", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["chrome", "webDriverBiDi"], - "expectations": ["PASS"] - }, { "testIdPattern": "[emulation.spec] Emulation Page.viewport should detect touch when applying viewport with touches", "platforms": ["darwin", "linux", "win32"], "parameters": ["firefox", "webDriverBiDi"], "expectations": ["FAIL"] }, - { - "testIdPattern": "[emulation.spec] Emulation Page.viewport should get the proper viewport size", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["chrome", "webDriverBiDi"], - "expectations": ["PASS"] - }, { "testIdPattern": "[emulation.spec] Emulation Page.viewport should load correct pictures when emulation dpr", "platforms": ["darwin", "linux", "win32"], @@ -2148,12 +2034,6 @@ "parameters": ["firefox", "webDriverBiDi"], "expectations": ["FAIL"] }, - { - "testIdPattern": "[emulation.spec] Emulation Page.viewport should support touch emulation", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["chrome", "webDriverBiDi"], - "expectations": ["PASS"] - }, { "testIdPattern": "[emulation.spec] Emulation Page.viewport should support touch emulation", "platforms": ["darwin", "linux", "win32"], @@ -2407,12 +2287,6 @@ "parameters": ["chrome", "webDriverBiDi"], "expectations": ["PASS"] }, - { - "testIdPattern": "[ignorehttpserrors.spec] ignoreHTTPSErrors should work", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["chrome", "webDriverBiDi"], - "expectations": ["PASS"] - }, { "testIdPattern": "[ignorehttpserrors.spec] ignoreHTTPSErrors should work with mixed content", "platforms": ["darwin", "linux", "win32"], @@ -2425,12 +2299,6 @@ "parameters": ["chrome", "webDriverBiDi"], "expectations": ["PASS"] }, - { - "testIdPattern": "[ignorehttpserrors.spec] ignoreHTTPSErrors should work with mixed content", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["chrome", "webDriverBiDi"], - "expectations": ["PASS"] - }, { "testIdPattern": "[ignorehttpserrors.spec] ignoreHTTPSErrors should work with request interception", "platforms": ["darwin", "linux", "win32"], @@ -2791,12 +2659,6 @@ "parameters": ["chrome", "webDriverBiDi"], "expectations": ["PASS"] }, - { - "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Browser.close should terminate network waiters", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["chrome", "webDriverBiDi"], - "expectations": ["PASS"] - }, { "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Browser.close should terminate network waiters", "platforms": ["darwin", "linux", "win32"], @@ -4403,12 +4265,6 @@ "parameters": ["firefox", "webDriverBiDi"], "expectations": ["SKIP"] }, - { - "testIdPattern": "[target.spec] Target should create a worker from a shared worker", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["chrome", "webDriverBiDi"], - "expectations": ["SKIP"] - }, { "testIdPattern": "[target.spec] Target should have an opener", "platforms": ["darwin", "linux", "win32"], diff --git a/tools/sort-test-expectations.mjs b/tools/sort-test-expectations.mjs index e826fcd58d7..76868db1d69 100644 --- a/tools/sort-test-expectations.mjs +++ b/tools/sort-test-expectations.mjs @@ -13,9 +13,22 @@ import prettier from 'prettier'; const __dirname = url.fileURLToPath(new URL('.', import.meta.url)); const source = 'test/TestExpectations.json'; -const testExpectations = JSON.parse(fs.readFileSync(source, 'utf-8')); +let testExpectations = JSON.parse(fs.readFileSync(source, 'utf-8')); const committedExpectations = structuredClone(testExpectations); +function testIdMatchesExpectationPattern(title, pattern) { + const patternRegExString = pattern + // Replace `*` with non special character + .replace(/\*/g, '--STAR--') + // Escape special characters https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions#escaping + .replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + // Replace placeholder with greedy match + .replace(/--STAR--/g, '(.*)?'); + // Match beginning and end explicitly + const patternRegEx = new RegExp(`^${patternRegExString}$`); + return patternRegEx.test(title); +} + const prettierConfig = await import( path.join(__dirname, '..', '.prettierrc.cjs') ); @@ -49,6 +62,52 @@ testExpectations.forEach(item => { } }); +function isSubset(superset, subset) { + let isSubset = true; + + for (const p of subset) { + if (!superset.has(p)) { + isSubset = false; + } + } + + return isSubset; +} + +const toBeRemoved = new Set(); +for (let i = testExpectations.length - 1; i >= 0; i--) { + const expectation = testExpectations[i]; + const params = new Set(expectation.parameters); + const labels = new Set(expectation.expectations); + const platforms = new Set(expectation.platforms); + + for (let j = i - 1; j >= 0; j--) { + const candidate = testExpectations[j]; + const candidateParams = new Set(candidate.parameters); + const candidateLabels = new Set(candidate.expectations); + const candidatePlatforms = new Set(candidate.platforms); + + if ( + testIdMatchesExpectationPattern( + expectation.testIdPattern, + candidate.testIdPattern + ) && + isSubset(candidateParams, params) && + isSubset(candidatePlatforms, platforms) + ) { + if (isSubset(candidateLabels, labels)) { + console.log('removing', expectation, 'already covered by', candidate); + toBeRemoved.add(expectation); + } + break; + } + } +} + +testExpectations = testExpectations.filter(item => { + return !toBeRemoved.has(item); +}); + if (process.argv.includes('--lint')) { if ( JSON.stringify(committedExpectations) !== JSON.stringify(testExpectations)