From c16a4a72768170b085b0eb49116138ad4b076ba7 Mon Sep 17 00:00:00 2001
From: Alex Rudenko <OrKoN@users.noreply.github.com>
Date: Tue, 20 Jun 2023 13:56:11 +0200
Subject: [PATCH] ci: check number of tests (#10423)

---
 package.json                   |  2 +-
 test/TestExpectations.json     | 16 ++++++++--------
 tools/mochaRunner/src/main.ts  | 15 +++++++++++++++
 tools/mochaRunner/src/types.ts |  2 +-
 4 files changed, 25 insertions(+), 10 deletions(-)

diff --git a/package.json b/package.json
index 4b76a15ae77..8ed8caff6b8 100644
--- a/package.json
+++ b/package.json
@@ -34,7 +34,7 @@
     "test:firefox:headful": "wireit",
     "test:firefox:headless": "wireit",
     "test:firefox": "wireit",
-    "test": "cross-env PUPPETEER_DEFERRED_PROMISE_DEBUG_TIMEOUT=20000 node tools/mochaRunner/lib/main.js",
+    "test": "cross-env PUPPETEER_DEFERRED_PROMISE_DEBUG_TIMEOUT=20000 node tools/mochaRunner/lib/main.js --min-tests 1052",
     "validate-licenses": "tsx tools/third_party/validate-licenses.ts"
   },
   "wireit": {
diff --git a/test/TestExpectations.json b/test/TestExpectations.json
index f08febaa58f..7c8f6595f29 100644
--- a/test/TestExpectations.json
+++ b/test/TestExpectations.json
@@ -1401,13 +1401,13 @@
     "testIdPattern": "[browser.spec] Browser specs Browser.isConnected should set the browser connected state",
     "platforms": ["darwin", "linux", "win32"],
     "parameters": ["firefox", "webDriverBiDi"],
-    "expectations": ["FAIL"]
+    "expectations": ["SKIP"]
   },
   {
     "testIdPattern": "[browser.spec] Browser specs Browser.process should not return child_process for remote browser",
     "platforms": ["darwin", "linux", "win32"],
     "parameters": ["firefox", "webDriverBiDi"],
-    "expectations": ["FAIL"]
+    "expectations": ["SKIP"]
   },
   {
     "testIdPattern": "[browser.spec] Browser specs Browser.version should return version",
@@ -1467,13 +1467,13 @@
     "testIdPattern": "[chromiumonly.spec] Chromium-Specific Launcher tests Puppeteer.launch |browserURL| option should be able to connect using browserUrl, with and without trailing slash",
     "platforms": ["darwin", "linux", "win32"],
     "parameters": ["firefox", "webDriverBiDi"],
-    "expectations": ["FAIL", "PASS"]
+    "expectations": ["SKIP"]
   },
   {
     "testIdPattern": "[click.spec] Page.click should click offscreen buttons",
     "platforms": ["darwin", "linux", "win32"],
     "parameters": ["firefox", "webDriverBiDi"],
-    "expectations": ["FAIL", "TIMEOUT"]
+    "expectations": ["SKIP"]
   },
   {
     "testIdPattern": "[click.spec] Page.click should click on checkbox label and toggle",
@@ -2139,13 +2139,13 @@
     "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.launch tmp profile should be cleaned up",
     "platforms": ["darwin", "linux", "win32"],
     "parameters": ["firefox", "webDriverBiDi"],
-    "expectations": ["FAIL"]
+    "expectations": ["SKIP"]
   },
   {
     "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.launch userDataDir option should restore cookies",
     "platforms": ["darwin", "linux", "win32"],
     "parameters": ["firefox", "webDriverBiDi"],
-    "expectations": ["FAIL", "PASS", "TIMEOUT"]
+    "expectations": ["SKIP"]
   },
   {
     "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.launch userDataDir option should restore state",
@@ -2253,13 +2253,13 @@
     "testIdPattern": "[navigation.spec] navigation Page.goto should navigate to dataURL and fire dataURL requests",
     "platforms": ["darwin", "linux", "win32"],
     "parameters": ["firefox", "webDriverBiDi"],
-    "expectations": ["FAIL"]
+    "expectations": ["SKIP"]
   },
   {
     "testIdPattern": "[navigation.spec] navigation Page.goto should navigate to empty page with domcontentloaded",
     "platforms": ["darwin", "linux", "win32"],
     "parameters": ["firefox", "webDriverBiDi"],
-    "expectations": ["FAIL"]
+    "expectations": ["SKIP"]
   },
   {
     "testIdPattern": "[navigation.spec] navigation Page.goto should navigate to empty page with networkidle0",
diff --git a/tools/mochaRunner/src/main.ts b/tools/mochaRunner/src/main.ts
index 5c91a49552d..1487ea3aa62 100644
--- a/tools/mochaRunner/src/main.ts
+++ b/tools/mochaRunner/src/main.ts
@@ -85,6 +85,12 @@ async function main() {
     }
   }
 
+  const minTestsIdx = process.argv.indexOf('--min-tests');
+  let minTests = 0;
+  if (minTestsIdx !== -1) {
+    minTests = Number(process.argv[minTestsIdx + 1]);
+  }
+
   const platform = zPlatform.parse(os.platform());
 
   const expectations = readJSON(
@@ -211,6 +217,7 @@ async function main() {
           platforms: [os.platform()],
           parameters,
         });
+        const totalTests = results.stats.tests;
         results.parameters = parameters;
         results.platform = platform;
         results.date = new Date().toISOString();
@@ -220,6 +227,14 @@ async function main() {
           results.updates = updates;
           writeJSON(tmpFilename, results);
         } else {
+          if (totalTests < minTests) {
+            fail = true;
+            console.log(
+              `Test run matches expectations but the number of discovered tests is too low (expected: ${minTests}, actual: ${totalTests}).`
+            );
+            writeJSON(tmpFilename, results);
+            continue;
+          }
           console.log('Test run matches expectations');
           writeJSON(tmpFilename, results);
           continue;
diff --git a/tools/mochaRunner/src/types.ts b/tools/mochaRunner/src/types.ts
index 8d8a08ee983..e690c5df8df 100644
--- a/tools/mochaRunner/src/types.ts
+++ b/tools/mochaRunner/src/types.ts
@@ -55,7 +55,7 @@ export type MochaTestResult = {
 };
 
 export type MochaResults = {
-  stats: unknown;
+  stats: {tests: number};
   pending: MochaTestResult[];
   passes: MochaTestResult[];
   failures: MochaTestResult[];