chore: improve recommendations by the test runner (#9724)

tools/mochaRunner/README.md

@@ -5,20 +5,20 @@ Mocha Runner is a test runner on top of mocha. It uses `/test/TestSuites.json` a
 ## Running tests for Mocha Runner itself.
 
 ```
-npm run build:test && npx c8 node tools/mochaRunner/lib/test.js
+npm run build && npx c8 node tools/mochaRunner/lib/test.js
 ```
 
 ## Running tests using Mocha Runner
 
 ```
-npm run build:test && npm run test
+npm run build && npm run test
 ```
 
 By default, the runner runs all test suites applicable to the current platform.
 To pick a test suite, provide the `--test-suite` arguments. For example,
 
 ```
-npm run build:test && npm run test -- --test-suite chrome-headless
+npm run build && npm run test -- --test-suite chrome-headless
 ```
 
 ## TestSuites.json

tools/mochaRunner/src/test.ts

@@ -18,7 +18,11 @@ import assert from 'assert/strict';
 import test from 'node:test';
 
 import {TestExpectation} from './types.js';
-import {filterByParameters, getTestResultForFailure} from './utils.js';
+import {
+  filterByParameters,
+  getTestResultForFailure,
+  isWildCardPattern,
+} from './utils.js';
 import {getFilename, extendProcessEnv} from './utils.js';
 
 test('extendProcessEnv', () => {
@@ -61,3 +65,12 @@ test('filterByParameters', () => {
   );
   assert.equal(filterByParameters(expectations, ['other']).length, 0);
 });
+
+test('isWildCardPattern', () => {
+  assert.equal(isWildCardPattern(''), true);
+  assert.equal(isWildCardPattern('a'), false);
+  assert.equal(isWildCardPattern('[queryHandler.spec]'), true);
+  assert.equal(isWildCardPattern('[queryHandler.spec] '), true);
+  assert.equal(isWildCardPattern(' [queryHandler.spec] '), true);
+  assert.equal(isWildCardPattern('[queryHandler.spec] test'), false);
+});

tools/mochaRunner/src/utils.ts

@@ -94,6 +94,14 @@ type RecommendedExpecation = {
   action: 'remove' | 'add' | 'update';
 };
 
+export function isWildCardPattern(testIdPattern: string): boolean {
+  testIdPattern = testIdPattern.trim();
+  return (
+    testIdPattern === '' ||
+    Boolean(testIdPattern.match(/^\[[a-zA-Z]+\.spec\]$/))
+  );
+}
+
 export function getExpectationUpdates(
   results: MochaResults,
   expecations: TestExpectation[],
@@ -105,10 +113,10 @@ export function getExpectationUpdates(
   const output: RecommendedExpecation[] = [];
 
   for (const pass of results.passes) {
-    const expectation = findEffectiveExpectationForTest(expecations, pass);
-    if (expectation && !expectation.expectations.includes('PASS')) {
+    const expectationEntry = findEffectiveExpectationForTest(expecations, pass);
+    if (expectationEntry && !expectationEntry.expectations.includes('PASS')) {
       output.push({
-        expectation,
+        expectation: expectationEntry,
         test: pass,
         action: 'remove',
       });
@@ -116,16 +124,27 @@ export function getExpectationUpdates(
   }
 
   for (const failure of results.failures) {
-    const expectation = findEffectiveExpectationForTest(expecations, failure);
-    if (expectation) {
+    const expectationEntry = findEffectiveExpectationForTest(
+      expecations,
+      failure
+    );
+    // If the effective explanation is a wildcard, we recommend adding a new
+    // expectation instead of updating the wildcard that might affect multiple
+    // tests.
+    if (
+      expectationEntry &&
+      !isWildCardPattern(expectationEntry.testIdPattern)
+    ) {
       if (
-        !expectation.expectations.includes(getTestResultForFailure(failure))
+        !expectationEntry.expectations.includes(
+          getTestResultForFailure(failure)
+        )
       ) {
         output.push({
           expectation: {
-            ...expectation,
+            ...expectationEntry,
             expectations: [
-              ...expectation.expectations,
+              ...expectationEntry.expectations,
               getTestResultForFailure(failure),
             ],
           },