diff --git a/tools/mochaRunner/README.md b/tools/mochaRunner/README.md index b3a8146c..fd87ec19 100644 --- a/tools/mochaRunner/README.md +++ b/tools/mochaRunner/README.md @@ -5,20 +5,20 @@ Mocha Runner is a test runner on top of mocha. It uses `/test/TestSuites.json` a ## Running tests for Mocha Runner itself. ``` -npm run build:test && npx c8 node tools/mochaRunner/lib/test.js +npm run build && npx c8 node tools/mochaRunner/lib/test.js ``` ## Running tests using Mocha Runner ``` -npm run build:test && npm run test +npm run build && npm run test ``` By default, the runner runs all test suites applicable to the current platform. To pick a test suite, provide the `--test-suite` arguments. For example, ``` -npm run build:test && npm run test -- --test-suite chrome-headless +npm run build && npm run test -- --test-suite chrome-headless ``` ## TestSuites.json diff --git a/tools/mochaRunner/src/test.ts b/tools/mochaRunner/src/test.ts index b9b93e8e..7e880488 100644 --- a/tools/mochaRunner/src/test.ts +++ b/tools/mochaRunner/src/test.ts @@ -18,7 +18,11 @@ import assert from 'assert/strict'; import test from 'node:test'; import {TestExpectation} from './types.js'; -import {filterByParameters, getTestResultForFailure} from './utils.js'; +import { + filterByParameters, + getTestResultForFailure, + isWildCardPattern, +} from './utils.js'; import {getFilename, extendProcessEnv} from './utils.js'; test('extendProcessEnv', () => { @@ -61,3 +65,12 @@ test('filterByParameters', () => { ); assert.equal(filterByParameters(expectations, ['other']).length, 0); }); + +test('isWildCardPattern', () => { + assert.equal(isWildCardPattern(''), true); + assert.equal(isWildCardPattern('a'), false); + assert.equal(isWildCardPattern('[queryHandler.spec]'), true); + assert.equal(isWildCardPattern('[queryHandler.spec] '), true); + assert.equal(isWildCardPattern(' [queryHandler.spec] '), true); + assert.equal(isWildCardPattern('[queryHandler.spec] test'), false); +}); diff --git a/tools/mochaRunner/src/utils.ts b/tools/mochaRunner/src/utils.ts index 4b050273..687b6d34 100644 --- a/tools/mochaRunner/src/utils.ts +++ b/tools/mochaRunner/src/utils.ts @@ -94,6 +94,14 @@ type RecommendedExpecation = { action: 'remove' | 'add' | 'update'; }; +export function isWildCardPattern(testIdPattern: string): boolean { + testIdPattern = testIdPattern.trim(); + return ( + testIdPattern === '' || + Boolean(testIdPattern.match(/^\[[a-zA-Z]+\.spec\]$/)) + ); +} + export function getExpectationUpdates( results: MochaResults, expecations: TestExpectation[], @@ -105,10 +113,10 @@ export function getExpectationUpdates( const output: RecommendedExpecation[] = []; for (const pass of results.passes) { - const expectation = findEffectiveExpectationForTest(expecations, pass); - if (expectation && !expectation.expectations.includes('PASS')) { + const expectationEntry = findEffectiveExpectationForTest(expecations, pass); + if (expectationEntry && !expectationEntry.expectations.includes('PASS')) { output.push({ - expectation, + expectation: expectationEntry, test: pass, action: 'remove', }); @@ -116,16 +124,27 @@ export function getExpectationUpdates( } for (const failure of results.failures) { - const expectation = findEffectiveExpectationForTest(expecations, failure); - if (expectation) { + const expectationEntry = findEffectiveExpectationForTest( + expecations, + failure + ); + // If the effective explanation is a wildcard, we recommend adding a new + // expectation instead of updating the wildcard that might affect multiple + // tests. + if ( + expectationEntry && + !isWildCardPattern(expectationEntry.testIdPattern) + ) { if ( - !expectation.expectations.includes(getTestResultForFailure(failure)) + !expectationEntry.expectations.includes( + getTestResultForFailure(failure) + ) ) { output.push({ expectation: { - ...expectation, + ...expectationEntry, expectations: [ - ...expectation.expectations, + ...expectationEntry.expectations, getTestResultForFailure(failure), ], },