build: auto-remove duplicate expectations (#11922)

test/TestExpectations.json

@@ -173,12 +173,6 @@
     "parameters": ["webDriverBiDi"],
     "expectations": ["PASS"]
   },
-  {
-    "testIdPattern": "[navigation.spec] navigation Page.goBack *",
-    "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["webDriverBiDi"],
-    "expectations": ["PASS"]
-  },
   {
     "testIdPattern": "[network.spec] network *",
     "platforms": ["darwin", "linux", "win32"],
@@ -587,12 +581,6 @@
     "parameters": ["firefox"],
     "expectations": ["FAIL", "PASS"]
   },
-  {
-    "testIdPattern": "[elementhandle.spec] ElementHandle specs ElementHandle.click should return Point data",
-    "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["webDriverBiDi"],
-    "expectations": ["PASS"]
-  },
   {
     "testIdPattern": "[elementhandle.spec] ElementHandle specs ElementHandle.clickablePoint should not work if the click box is not visible due to the iframe",
     "platforms": ["darwin", "linux", "win32"],
@@ -605,18 +593,6 @@
     "parameters": ["chrome", "webDriverBiDi"],
     "expectations": ["PASS"]
   },
-  {
-    "testIdPattern": "[emulation.spec] Emulation Page.viewport should get the proper viewport size",
-    "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["webDriverBiDi"],
-    "expectations": ["PASS"]
-  },
-  {
-    "testIdPattern": "[emulation.spec] Emulation Page.viewport should support mobile emulation",
-    "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["webDriverBiDi"],
-    "expectations": ["PASS"]
-  },
   {
     "testIdPattern": "[evaluation.spec] Evaluation specs Page.evaluate should replace symbols with undefined",
     "platforms": ["darwin", "linux", "win32"],
@@ -720,12 +696,6 @@
     "parameters": ["webDriverBiDi"],
     "expectations": ["FAIL"]
   },
-  {
-    "testIdPattern": "[jshandle.spec] JSHandle Page.evaluateHandle should use the same JS wrappers",
-    "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["webDriverBiDi"],
-    "expectations": ["PASS"]
-  },
   {
     "testIdPattern": "[keyboard.spec] Keyboard should send a character with sendCharacter in iframe",
     "platforms": ["darwin", "linux", "win32"],
@@ -1010,12 +980,6 @@
     "parameters": ["webDriverBiDi"],
     "expectations": ["SKIP"]
   },
-  {
-    "testIdPattern": "[page.spec] Page Page.Events.Console should work for different console API calls with logging functions",
-    "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["webDriverBiDi"],
-    "expectations": ["PASS"]
-  },
   {
     "testIdPattern": "[page.spec] Page Page.Events.error should throw when page crashes",
     "platforms": ["darwin", "linux", "win32"],
@@ -1083,12 +1047,6 @@
     "parameters": ["firefox", "webDriverBiDi"],
     "expectations": ["FAIL"]
   },
-  {
-    "testIdPattern": "[page.spec] Page Page.waitForNetworkIdle should work with aborted requests",
-    "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["webDriverBiDi"],
-    "expectations": ["PASS"]
-  },
   {
     "testIdPattern": "[pdf.spec] Page.pdf *",
     "platforms": ["darwin", "linux", "win32"],
@@ -1149,18 +1107,6 @@
     "parameters": ["webDriverBiDi"],
     "expectations": ["SKIP"]
   },
-  {
-    "testIdPattern": "[queryhandler.spec] Query handler tests P selectors should work with :hover",
-    "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["webDriverBiDi"],
-    "expectations": ["PASS"]
-  },
-  {
-    "testIdPattern": "[queryhandler.spec] Query handler tests Text selectors in Page should clear caches",
-    "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["webDriverBiDi"],
-    "expectations": ["PASS"]
-  },
   {
     "testIdPattern": "[queryObjects.spec] *",
     "platforms": ["darwin", "linux", "win32"],
@@ -1325,12 +1271,6 @@
     "parameters": ["firefox", "webDriverBiDi"],
     "expectations": ["SKIP"]
   },
-  {
-    "testIdPattern": "[accessibility.spec] Accessibility filtering children of leaf nodes rich text editable fields should have children",
-    "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["chrome", "webDriverBiDi"],
-    "expectations": ["PASS"]
-  },
   {
     "testIdPattern": "[accessibility.spec] Accessibility get snapshots while the tree is re-calculated",
     "platforms": ["darwin", "linux", "win32"],
@@ -1403,12 +1343,6 @@
     "parameters": ["chrome", "webDriverBiDi"],
     "expectations": ["PASS"]
   },
-  {
-    "testIdPattern": "[browser.spec] Browser specs Browser.isConnected should set the browser connected state",
-    "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["chrome", "webDriverBiDi"],
-    "expectations": ["PASS"]
-  },
   {
     "testIdPattern": "[browser.spec] Browser specs Browser.process should not return child_process for remote browser",
     "platforms": ["darwin", "linux", "win32"],
@@ -1487,18 +1421,6 @@
     "parameters": ["cdp", "firefox"],
     "expectations": ["FAIL"]
   },
-  {
-    "testIdPattern": "[browsercontext.spec] BrowserContext should close all belonging targets once closing context",
-    "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["chrome", "webDriverBiDi"],
-    "expectations": ["PASS"]
-  },
-  {
-    "testIdPattern": "[browsercontext.spec] BrowserContext should create new incognito context",
-    "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["chrome", "webDriverBiDi"],
-    "expectations": ["PASS"]
-  },
   {
     "testIdPattern": "[browsercontext.spec] BrowserContext should create new incognito context",
     "platforms": ["darwin", "linux", "win32"],
@@ -1524,24 +1446,12 @@
     "parameters": ["firefox", "webDriverBiDi"],
     "expectations": ["PASS"]
   },
-  {
-    "testIdPattern": "[browsercontext.spec] BrowserContext should timeout waiting for a non-existent target",
-    "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["chrome", "webDriverBiDi"],
-    "expectations": ["PASS"]
-  },
   {
     "testIdPattern": "[browsercontext.spec] BrowserContext should wait for a target",
     "platforms": ["darwin", "linux", "win32"],
     "parameters": ["cdp", "firefox"],
     "expectations": ["SKIP"]
   },
-  {
-    "testIdPattern": "[browsercontext.spec] BrowserContext should wait for a target",
-    "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["chrome", "webDriverBiDi"],
-    "expectations": ["PASS"]
-  },
   {
     "testIdPattern": "[browsercontext.spec] BrowserContext should work across sessions",
     "platforms": ["darwin", "linux", "win32"],
@@ -1842,18 +1752,6 @@
     "parameters": ["cdp", "firefox"],
     "expectations": ["FAIL"]
   },
-  {
-    "testIdPattern": "[coverage.spec] Coverage specs CSSCoverage should work with complicated usecases",
-    "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["chrome", "webDriverBiDi"],
-    "expectations": ["PASS"]
-  },
-  {
-    "testIdPattern": "[coverage.spec] Coverage specs JSCoverage should not ignore eval() scripts if reportAnonymousScripts is true",
-    "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["chrome", "webDriverBiDi"],
-    "expectations": ["PASS"]
-  },
   {
     "testIdPattern": "[debugInfo.spec] DebugInfo Browser.debugInfo should work",
     "platforms": ["darwin", "linux", "win32"],
@@ -2112,24 +2010,12 @@
     "parameters": ["firefox", "webDriverBiDi"],
     "expectations": ["FAIL"]
   },
-  {
-    "testIdPattern": "[emulation.spec] Emulation Page.viewport should detect touch when applying viewport with touches",
-    "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["chrome", "webDriverBiDi"],
-    "expectations": ["PASS"]
-  },
   {
     "testIdPattern": "[emulation.spec] Emulation Page.viewport should detect touch when applying viewport with touches",
     "platforms": ["darwin", "linux", "win32"],
     "parameters": ["firefox", "webDriverBiDi"],
     "expectations": ["FAIL"]
   },
-  {
-    "testIdPattern": "[emulation.spec] Emulation Page.viewport should get the proper viewport size",
-    "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["chrome", "webDriverBiDi"],
-    "expectations": ["PASS"]
-  },
   {
     "testIdPattern": "[emulation.spec] Emulation Page.viewport should load correct pictures when emulation dpr",
     "platforms": ["darwin", "linux", "win32"],
@@ -2148,12 +2034,6 @@
     "parameters": ["firefox", "webDriverBiDi"],
     "expectations": ["FAIL"]
   },
-  {
-    "testIdPattern": "[emulation.spec] Emulation Page.viewport should support touch emulation",
-    "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["chrome", "webDriverBiDi"],
-    "expectations": ["PASS"]
-  },
   {
     "testIdPattern": "[emulation.spec] Emulation Page.viewport should support touch emulation",
     "platforms": ["darwin", "linux", "win32"],
@@ -2407,12 +2287,6 @@
     "parameters": ["chrome", "webDriverBiDi"],
     "expectations": ["PASS"]
   },
-  {
-    "testIdPattern": "[ignorehttpserrors.spec] ignoreHTTPSErrors should work",
-    "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["chrome", "webDriverBiDi"],
-    "expectations": ["PASS"]
-  },
   {
     "testIdPattern": "[ignorehttpserrors.spec] ignoreHTTPSErrors should work with mixed content",
     "platforms": ["darwin", "linux", "win32"],
@@ -2425,12 +2299,6 @@
     "parameters": ["chrome", "webDriverBiDi"],
     "expectations": ["PASS"]
   },
-  {
-    "testIdPattern": "[ignorehttpserrors.spec] ignoreHTTPSErrors should work with mixed content",
-    "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["chrome", "webDriverBiDi"],
-    "expectations": ["PASS"]
-  },
   {
     "testIdPattern": "[ignorehttpserrors.spec] ignoreHTTPSErrors should work with request interception",
     "platforms": ["darwin", "linux", "win32"],
@@ -2791,12 +2659,6 @@
     "parameters": ["chrome", "webDriverBiDi"],
     "expectations": ["PASS"]
   },
-  {
-    "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Browser.close should terminate network waiters",
-    "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["chrome", "webDriverBiDi"],
-    "expectations": ["PASS"]
-  },
   {
     "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Browser.close should terminate network waiters",
     "platforms": ["darwin", "linux", "win32"],
@@ -4403,12 +4265,6 @@
     "parameters": ["firefox", "webDriverBiDi"],
     "expectations": ["SKIP"]
   },
-  {
-    "testIdPattern": "[target.spec] Target should create a worker from a shared worker",
-    "platforms": ["darwin", "linux", "win32"],
-    "parameters": ["chrome", "webDriverBiDi"],
-    "expectations": ["SKIP"]
-  },
   {
     "testIdPattern": "[target.spec] Target should have an opener",
     "platforms": ["darwin", "linux", "win32"],

tools/sort-test-expectations.mjs

@@ -13,9 +13,22 @@ import prettier from 'prettier';
 
 const __dirname = url.fileURLToPath(new URL('.', import.meta.url));
 const source = 'test/TestExpectations.json';
-const testExpectations = JSON.parse(fs.readFileSync(source, 'utf-8'));
+let testExpectations = JSON.parse(fs.readFileSync(source, 'utf-8'));
 const committedExpectations = structuredClone(testExpectations);
 
+function testIdMatchesExpectationPattern(title, pattern) {
+  const patternRegExString = pattern
+    // Replace `*` with non special character
+    .replace(/\*/g, '--STAR--')
+    // Escape special characters https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions#escaping
+    .replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
+    // Replace placeholder with greedy match
+    .replace(/--STAR--/g, '(.*)?');
+  // Match beginning and end explicitly
+  const patternRegEx = new RegExp(`^${patternRegExString}$`);
+  return patternRegEx.test(title);
+}
+
 const prettierConfig = await import(
   path.join(__dirname, '..', '.prettierrc.cjs')
 );
@@ -49,6 +62,52 @@ testExpectations.forEach(item => {
   }
 });
 
+function isSubset(superset, subset) {
+  let isSubset = true;
+
+  for (const p of subset) {
+    if (!superset.has(p)) {
+      isSubset = false;
+    }
+  }
+
+  return isSubset;
+}
+
+const toBeRemoved = new Set();
+for (let i = testExpectations.length - 1; i >= 0; i--) {
+  const expectation = testExpectations[i];
+  const params = new Set(expectation.parameters);
+  const labels = new Set(expectation.expectations);
+  const platforms = new Set(expectation.platforms);
+
+  for (let j = i - 1; j >= 0; j--) {
+    const candidate = testExpectations[j];
+    const candidateParams = new Set(candidate.parameters);
+    const candidateLabels = new Set(candidate.expectations);
+    const candidatePlatforms = new Set(candidate.platforms);
+
+    if (
+      testIdMatchesExpectationPattern(
+        expectation.testIdPattern,
+        candidate.testIdPattern
+      ) &&
+      isSubset(candidateParams, params) &&
+      isSubset(candidatePlatforms, platforms)
+    ) {
+      if (isSubset(candidateLabels, labels)) {
+        console.log('removing', expectation, 'already covered by', candidate);
+        toBeRemoved.add(expectation);
+      }
+      break;
+    }
+  }
+}
+
+testExpectations = testExpectations.filter(item => {
+  return !toBeRemoved.has(item);
+});
+
 if (process.argv.includes('--lint')) {
   if (
     JSON.stringify(committedExpectations) !== JSON.stringify(testExpectations)