ci: check number of tests (#10423)

package.json

@@ -34,7 +34,7 @@
     "test:firefox:headful": "wireit",
     "test:firefox:headless": "wireit",
     "test:firefox": "wireit",
-    "test": "cross-env PUPPETEER_DEFERRED_PROMISE_DEBUG_TIMEOUT=20000 node tools/mochaRunner/lib/main.js",
+    "test": "cross-env PUPPETEER_DEFERRED_PROMISE_DEBUG_TIMEOUT=20000 node tools/mochaRunner/lib/main.js --min-tests 1052",
     "validate-licenses": "tsx tools/third_party/validate-licenses.ts"
   },
   "wireit": {

test/TestExpectations.json

@@ -1401,13 +1401,13 @@
     "testIdPattern": "[browser.spec] Browser specs Browser.isConnected should set the browser connected state",
     "platforms": ["darwin", "linux", "win32"],
     "parameters": ["firefox", "webDriverBiDi"],
-    "expectations": ["FAIL"]
+    "expectations": ["SKIP"]
   },
   {
     "testIdPattern": "[browser.spec] Browser specs Browser.process should not return child_process for remote browser",
     "platforms": ["darwin", "linux", "win32"],
     "parameters": ["firefox", "webDriverBiDi"],
-    "expectations": ["FAIL"]
+    "expectations": ["SKIP"]
   },
   {
     "testIdPattern": "[browser.spec] Browser specs Browser.version should return version",
@@ -1467,13 +1467,13 @@
     "testIdPattern": "[chromiumonly.spec] Chromium-Specific Launcher tests Puppeteer.launch |browserURL| option should be able to connect using browserUrl, with and without trailing slash",
     "platforms": ["darwin", "linux", "win32"],
     "parameters": ["firefox", "webDriverBiDi"],
-    "expectations": ["FAIL", "PASS"]
+    "expectations": ["SKIP"]
   },
   {
     "testIdPattern": "[click.spec] Page.click should click offscreen buttons",
     "platforms": ["darwin", "linux", "win32"],
     "parameters": ["firefox", "webDriverBiDi"],
-    "expectations": ["FAIL", "TIMEOUT"]
+    "expectations": ["SKIP"]
   },
   {
     "testIdPattern": "[click.spec] Page.click should click on checkbox label and toggle",
@@ -2139,13 +2139,13 @@
     "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.launch tmp profile should be cleaned up",
     "platforms": ["darwin", "linux", "win32"],
     "parameters": ["firefox", "webDriverBiDi"],
-    "expectations": ["FAIL"]
+    "expectations": ["SKIP"]
   },
   {
     "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.launch userDataDir option should restore cookies",
     "platforms": ["darwin", "linux", "win32"],
     "parameters": ["firefox", "webDriverBiDi"],
-    "expectations": ["FAIL", "PASS", "TIMEOUT"]
+    "expectations": ["SKIP"]
   },
   {
     "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.launch userDataDir option should restore state",
@@ -2253,13 +2253,13 @@
     "testIdPattern": "[navigation.spec] navigation Page.goto should navigate to dataURL and fire dataURL requests",
     "platforms": ["darwin", "linux", "win32"],
     "parameters": ["firefox", "webDriverBiDi"],
-    "expectations": ["FAIL"]
+    "expectations": ["SKIP"]
   },
   {
     "testIdPattern": "[navigation.spec] navigation Page.goto should navigate to empty page with domcontentloaded",
     "platforms": ["darwin", "linux", "win32"],
     "parameters": ["firefox", "webDriverBiDi"],
-    "expectations": ["FAIL"]
+    "expectations": ["SKIP"]
   },
   {
     "testIdPattern": "[navigation.spec] navigation Page.goto should navigate to empty page with networkidle0",

tools/mochaRunner/src/main.ts

@@ -85,6 +85,12 @@ async function main() {
     }
   }
 
+  const minTestsIdx = process.argv.indexOf('--min-tests');
+  let minTests = 0;
+  if (minTestsIdx !== -1) {
+    minTests = Number(process.argv[minTestsIdx + 1]);
+  }
+
   const platform = zPlatform.parse(os.platform());
 
   const expectations = readJSON(
@@ -211,6 +217,7 @@ async function main() {
           platforms: [os.platform()],
           parameters,
         });
+        const totalTests = results.stats.tests;
         results.parameters = parameters;
         results.platform = platform;
         results.date = new Date().toISOString();
@@ -220,6 +227,14 @@ async function main() {
           results.updates = updates;
           writeJSON(tmpFilename, results);
         } else {
+          if (totalTests < minTests) {
+            fail = true;
+            console.log(
+              `Test run matches expectations but the number of discovered tests is too low (expected: ${minTests}, actual: ${totalTests}).`
+            );
+            writeJSON(tmpFilename, results);
+            continue;
+          }
           console.log('Test run matches expectations');
           writeJSON(tmpFilename, results);
           continue;

tools/mochaRunner/src/types.ts

@@ -55,7 +55,7 @@ export type MochaTestResult = {
 };
 
 export type MochaResults = {
-  stats: unknown;
+  stats: {tests: number};
   pending: MochaTestResult[];
   passes: MochaTestResult[];
   failures: MochaTestResult[];