tests: drop jasmine test runner (#1519)

This patch introduces a tiny test runner to run puppeteer tests.
The test runner is self-container and allows parallel (wrt IO) test execution.
It will also allow us to split tests into multiple files if necessary.

Comparing to the jasmine, the testrunner supports parallel execution, properly
handles "unhandled promise rejection" event and signals.

Comparing to ava/jest, the testrunner doesn't run multiple node processes,
which makes it simpler but sufficient for our goals.

.eslintignore

@@ -1,4 +1,6 @@
 third_party/*
 utils/doclint/check_public_api/test/
+utils/testrunner/examples/
 node6/*
-node6-test/*
+node6-test/*
+node6-testrunner/*

.eslintrc.js

@@ -71,7 +71,7 @@ module.exports = {
         "no-unsafe-negation": 2,
         "radix": 2,
         "valid-typeof": 2,
-        "no-unused-vars": [2, { "args": "none", "vars": "local" }],
+        "no-unused-vars": [2, { "args": "none", "vars": "local", "varsIgnorePattern": "([fx]?describe|[fx]?it|beforeAll|beforeEach|afterAll|afterEach)" }],
         "no-implicit-globals": [2],
 
         // es2015 features

.gitignore

@@ -10,3 +10,4 @@
 package-lock.json
 /node6
 /node6-test
+/node6-testrunner

.travis.yml

@@ -20,7 +20,6 @@ script:
   - 'if [ "$NODE7" = "true" ]; then yarn run lint; fi'
   - 'if [ "$NODE7" = "true" ]; then yarn run coverage; fi'
   - 'if [ "$NODE7" = "true" ]; then yarn run test-doclint; fi'
-  - 'if [ "$NODE6" = "true" ]; then yarn run test-node6-transformer; fi'
   - 'if [ "$NODE6" = "true" ]; then yarn run build; fi'
   - 'if [ "$NODE6" = "true" ]; then yarn run unit-node6; fi'
 jobs:

CONTRIBUTING.md

@@ -111,7 +111,8 @@ A barrier for introducing new installation dependencies is especially high:
 - tests should work on all three platforms: Mac, Linux and Win. This is especially important for screenshot tests.
 
 Puppeteer tests are located in [test/test.js](https://github.com/GoogleChrome/puppeteer/blob/master/test/test.js)
-and are written using [Jasmine](https://jasmine.github.io/) testing framework. Despite being named 'unit', these are integration tests, making sure public API methods and events work as expected.
+and are written with a [TestRunner](https://github.com/GoogleChrome/puppeteer/tree/master/utils/testrunner) framework.
+Despite being named 'unit', these are integration tests, making sure public API methods and events work as expected.
 
 - To run all tests:
 ```

package.json

@@ -8,17 +8,17 @@
     "node": ">=6.4.0"
   },
   "scripts": {
-    "unit": "jasmine test/test.js",
+    "unit": "node test/test.js",
-    "debug-unit": "cross-env DEBUG_TEST=true node --inspect-brk ./node_modules/jasmine/bin/jasmine.js test/test.js",
+    "debug-unit": "cross-env DEBUG_TEST=true node --inspect-brk test/test.js",
-    "test-doclint": "jasmine utils/doclint/check_public_api/test/test.js && jasmine utils/doclint/preprocessor/test.js",
+    "test-doclint": "node utils/doclint/check_public_api/test/test.js && node utils/doclint/preprocessor/test.js",
     "test": "npm run lint --silent && npm run coverage && npm run test-doclint && npm run test-node6-transformer",
     "install": "node install.js",
     "lint": "([ \"$CI\" = true ] && eslint --quiet -f codeframe . || eslint .) && npm run tsc && npm run doc",
     "doc": "node utils/doclint/cli.js",
     "coverage": "cross-env COVERAGE=true npm run unit",
-    "test-node6-transformer": "jasmine utils/node6-transform/test/test.js",
+    "test-node6-transformer": "node utils/node6-transform/test/test.js",
     "build": "node utils/node6-transform/index.js",
-    "unit-node6": "jasmine node6-test/test.js",
+    "unit-node6": "node node6-test/test.js",
     "tsc": "tsc -p ."
   },
   "author": "The Chromium Authors",
@@ -47,7 +47,6 @@
     "cross-env": "^5.0.5",
     "eslint": "^4.0.0",
     "esprima": "^4.0.0",
-    "jasmine": "^2.6.0",
     "markdown-toc": "^1.1.0",
     "minimist": "^1.2.0",
     "ncp": "^2.0.0",

test/golden-utils.js

@@ -20,15 +20,7 @@ const mime = require('mime');
 const PNG = require('pngjs').PNG;
 const pixelmatch = require('pixelmatch');
 
-module.exports = {
+module.exports = {compare};
-  addMatchers: function(jasmine, goldenPath, outputPath) {
-    jasmine.addMatchers({
-      toBeGolden: function(util, customEqualityTesters) {
-        return { compare: compare.bind(null, goldenPath, outputPath) };
-      }
-    });
-  },
-};
 
 const GoldenComparators = {
   'image/png': compareImages,

test/test.js

@@ -13,7 +13,6 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
 const fs = require('fs');
 const rm = require('rimraf').sync;
 const path = require('path');
@@ -26,7 +25,6 @@ const SimpleServer = require('./server/SimpleServer');
 const GoldenUtils = require('./golden-utils');
 
 const YELLOW_COLOR = '\x1b[33m';
-const RED_COLOR = '\x1b[31m';
 const RESET_COLOR = '\x1b[0m';
 
 const GOLDEN_DIR = path.join(__dirname, 'golden');
@@ -52,11 +50,6 @@ const defaultBrowserOptions = {
   args: ['--no-sandbox']
 };
 
-if (process.env.DEBUG_TEST || slowMo)
-  jasmine.DEFAULT_TIMEOUT_INTERVAL = 1000 * 1000 * 1000;
-else
-  jasmine.DEFAULT_TIMEOUT_INTERVAL = 10 * 1000;
-
 // Make sure the `npm install` was run after the chromium roll.
 {
   const Downloader = require('../utils/ChromiumDownloader');
@@ -65,23 +58,18 @@ else
   console.assert(revisionInfo.downloaded, `Chromium r${chromiumRevision} is not downloaded. Run 'npm install' and try to re-run tests.`);
 }
 
-// Hack to get the currently-running spec name.
+const timeout = process.env.DEBUG_TEST || slowMo ?  0 : 10 * 1000;
-let specName = null;
-jasmine.getEnv().addReporter({
-  specStarted: result => specName = result.fullName
-});
 
-// Setup unhandledRejectionHandlers
+const {TestRunner, Reporter, Matchers}  = require('../utils/testrunner/');
-let hasUnhandledRejection = false;
+const runner = new TestRunner({timeout});
-process.on('unhandledRejection', error => {
+new Reporter(runner);
-  hasUnhandledRejection = true;
+
-  const textLines = [
+const {describe, xdescribe, fdescribe} = runner;
-    '',
+const {it, fit, xit} = runner;
-    `${RED_COLOR}[UNHANDLED PROMISE REJECTION]${RESET_COLOR} "${specName}"`,
+const {beforeAll, beforeEach, afterAll, afterEach} = runner;
-    error.stack,
+
-    '',
+const {expect} = new Matchers({
-  ];
+  toBeGolden: GoldenUtils.compare.bind(null, GOLDEN_DIR, OUTPUT_DIR)
-  console.error(textLines.join('\n'));
 });
 
 let server;
@@ -97,7 +85,6 @@ beforeAll(SX(async function() {
 beforeEach(SX(async function() {
   server.reset();
   httpsServer.reset();
-  GoldenUtils.addMatchers(jasmine, GOLDEN_DIR, OUTPUT_DIR);
 }));
 
 afterAll(SX(async function() {
@@ -3280,19 +3267,12 @@ describe('Page', function() {
       serverResponse.end();
       // Wait for the new page to load.
       await waitForEvents(newPage, 'load');
-
-      expect(hasUnhandledRejection).toBe(false);
-
       // Cleanup.
       await newPage.close();
     }));
   });
 });
 
-it('Unhandled promise rejections should not be thrown', function() {
-  expect(hasUnhandledRejection).toBe(false);
-});
-
 if (process.env.COVERAGE) {
   describe('COVERAGE', function(){
     const coverage = helper.publicAPICoverage();
@@ -3307,6 +3287,8 @@ if (process.env.COVERAGE) {
     }
   });
 }
+
+runner.run();
 /**
  * @param {!EventEmitter} emitter
  * @param {string} eventName
@@ -3361,8 +3343,7 @@ function cssPixelsToInches(px) {
   return px / 96;
 }
 
-// Since Jasmine doesn't like async functions, they should be wrapped
+// TODO: remove
-// in a SX function.
 function SX(fun) {
-  return done => Promise.resolve(fun()).then(done).catch(done.fail);
+  return fun;
 }

utils/doclint/check_public_api/test/test.js

@@ -14,8 +14,6 @@
  * limitations under the License.
  */
 
-const fs = require('fs');
-const rm = require('rimraf').sync;
 const path = require('path');
 const puppeteer = require('../../../..');
 const checkPublicAPI = require('..');
@@ -24,45 +22,48 @@ const mdBuilder = require('../MDBuilder');
 const jsBuilder = require('../JSBuilder');
 const GoldenUtils = require('../../../../test/golden-utils');
 
-const OUTPUT_DIR = path.join(__dirname, 'output');
+const {TestRunner, Reporter, Matchers}  = require('../../../testrunner/');
-const GOLDEN_DIR = path.join(__dirname, 'golden');
+const runner = new TestRunner();
+const reporter = new Reporter(runner);
+
+const {describe, xdescribe, fdescribe} = runner;
+const {it, fit, xit} = runner;
+const {beforeAll, beforeEach, afterAll, afterEach} = runner;
 
 let browser;
 let page;
-let specName;
 
-jasmine.getEnv().addReporter({
+beforeAll(async function() {
-  specStarted: result => specName = result.description
-});
-
-beforeAll(SX(async function() {
   browser = await puppeteer.launch({args: ['--no-sandbox']});
   page = await browser.newPage();
-  if (fs.existsSync(OUTPUT_DIR))
-    rm(OUTPUT_DIR);
-}));
-
-afterAll(SX(async function() {
-  await browser.close();
-}));
-
-describe('checkPublicAPI', function() {
-  it('diff-classes', SX(testLint));
-  it('diff-methods', SX(testLint));
-  it('diff-properties', SX(testLint));
-  it('diff-arguments', SX(testLint));
-  it('diff-events', SX(testLint));
-  it('check-duplicates', SX(testLint));
-  it('check-sorting', SX(testLint));
-  it('check-returns', SX(testLint));
-  it('js-builder-common', SX(testJSBuilder));
-  it('js-builder-inheritance', SX(testJSBuilder));
-  it('md-builder-common', SX(testMDBuilder));
 });
 
-async function testLint() {
+afterAll(async function() {
-  const dirPath = path.join(__dirname, specName);
+  await browser.close();
-  GoldenUtils.addMatchers(jasmine, dirPath, dirPath);
+});
+
+describe('checkPublicAPI', function() {
+  it('diff-classes', testLint);
+  it('diff-methods', testLint);
+  it('diff-properties', testLint);
+  it('diff-arguments', testLint);
+  it('diff-events', testLint);
+  it('check-duplicates', testLint);
+  it('check-sorting', testLint);
+  it('check-returns', testLint);
+  it('js-builder-common', testJSBuilder);
+  it('js-builder-inheritance', testJSBuilder);
+  it('md-builder-common', testMDBuilder);
+});
+
+runner.run();
+
+async function testLint(state, test) {
+  const dirPath = path.join(__dirname, test.name);
+  const {expect} = new Matchers({
+    toBeGolden: GoldenUtils.compare.bind(null, dirPath, dirPath)
+  });
+
   const factory = new SourceFactory();
   const mdSources = await factory.readdir(dirPath, '.md');
   const jsSources = await factory.readdir(dirPath, '.js');
@@ -71,18 +72,22 @@ async function testLint() {
   expect(errors.join('\n')).toBeGolden('result.txt');
 }
 
-async function testMDBuilder() {
+async function testMDBuilder(state, test) {
-  const dirPath = path.join(__dirname, specName);
+  const dirPath = path.join(__dirname, test.name);
-  GoldenUtils.addMatchers(jasmine, dirPath, dirPath);
+  const {expect} = new Matchers({
+    toBeGolden: GoldenUtils.compare.bind(null, dirPath, dirPath)
+  });
   const factory = new SourceFactory();
   const sources = await factory.readdir(dirPath, '.md');
   const {documentation} = await mdBuilder(page, sources);
   expect(serialize(documentation)).toBeGolden('result.txt');
 }
 
-async function testJSBuilder() {
+async function testJSBuilder(state, test) {
-  const dirPath = path.join(__dirname, specName);
+  const dirPath = path.join(__dirname, test.name);
-  GoldenUtils.addMatchers(jasmine, dirPath, dirPath);
+  const {expect} = new Matchers({
+    toBeGolden: GoldenUtils.compare.bind(null, dirPath, dirPath)
+  });
   const factory = new SourceFactory();
   const sources = await factory.readdir(dirPath, '.js');
   const {documentation} = await jsBuilder(sources);
@@ -110,8 +115,3 @@ function serialize(doc) {
   return JSON.stringify(result, null, 2);
 }
 
-// Since Jasmine doesn't like async functions, they should be wrapped
-// in a SX function.
-function SX(fun) {
-  return done => Promise.resolve(fun()).then(done).catch(done.fail);
-}

utils/doclint/preprocessor/test.js

@@ -19,6 +19,15 @@ const SourceFactory = require('../SourceFactory');
 const factory = new SourceFactory();
 const VERSION = require('../../../package.json').version;
 
+const {TestRunner, Reporter, Matchers}  = require('../../testrunner/');
+const runner = new TestRunner();
+new Reporter(runner);
+
+const {describe, xdescribe, fdescribe} = runner;
+const {it, fit, xit} = runner;
+const {beforeAll, beforeEach, afterAll, afterEach} = runner;
+const {expect} = new Matchers();
+
 describe('preprocessor', function() {
   it('should throw for unknown command', function() {
     const source = factory.createForTest('doc.md', getCommand('unknownCommand()'));
@@ -54,6 +63,8 @@ describe('preprocessor', function() {
   });
 });
 
+runner.run();
+
 function getCommand(name, body = '') {
   return `<!--gen:${name}-->${body}<!--gen:stop-->`;
 }

utils/node6-transform/index.js

@@ -21,7 +21,7 @@ const transformAsyncFunctions = require('./TransformAsyncFunctions');
 
 copyFolder(path.join(__dirname, '..', '..', 'lib'), path.join(__dirname, '..', '..', 'node6'));
 copyFolder(path.join(__dirname, '..', '..', 'test'), path.join(__dirname, '..', '..', 'node6-test'));
-
+copyFolder(path.join(__dirname, '..', '..', 'utils', 'testrunner'), path.join(__dirname, '..', '..', 'node6-testrunner'));
 
 function copyFolder(source, target) {
   if (fs.existsSync(target))
@@ -35,8 +35,11 @@ function copyFolder(source, target) {
       copyFolder(from, to);
     } else {
       let text = fs.readFileSync(from);
-      if (file.endsWith('.js'))
+      if (file.endsWith('.js')) {
-        text = transformAsyncFunctions(text.toString()).replace(/require\('\.\.\/lib\//g, `require('../node6/`);
+        text = transformAsyncFunctions(text.toString());
+        text = text.replace(/require\('\.\.\/lib\//g, `require('../node6/`);
+        text = text.replace(/require\('\.\.\/utils\/testrunner\//g, `require('../node6-testrunner/`);
+      }
       fs.writeFileSync(to, text);
     }
   });

utils/node6-transform/test/test.js

@@ -15,6 +15,16 @@
  */
 const transformAsyncFunctions = require('../TransformAsyncFunctions');
 
+const {TestRunner, Reporter, Matchers}  = require('../../testrunner/');
+const runner = new TestRunner();
+new Reporter(runner);
+
+const {describe, xdescribe, fdescribe} = runner;
+const {it, fit, xit} = runner;
+const {beforeAll, beforeEach, afterAll, afterEach} = runner;
+
+const {expect} = new Matchers();
+
 describe('TransformAsyncFunctions', function() {
   it('should convert a function expression', function(done) {
     const input = `(async function(){ return 123 })()`;
@@ -76,4 +86,6 @@ describe('TransformAsyncFunctions', function() {
     expect(output instanceof Promise).toBe(true);
     output.then(result => expect(result).toBe(123)).then(done);
   });
-});
+});
+
+runner.run();

utils/testrunner/Matchers.js

@@ -0,0 +1,99 @@
+/**
+ * Copyright 2017 Google Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+module.exports = class Matchers {
+  constructor(customMatchers = {}) {
+    this._matchers = {};
+    Object.assign(this._matchers, DefaultMatchers);
+    Object.assign(this._matchers, customMatchers);
+    this.expect = this.expect.bind(this);
+  }
+
+  addMatcher(name, matcher) {
+    this._matchers[name] = matcher;
+  }
+
+  expect(value) {
+    return new Expect(value, this._matchers);
+  }
+};
+
+class Expect {
+  constructor(value, matchers) {
+    this.not = {};
+    this.not.not = this;
+    for (const matcherName of Object.keys(matchers)) {
+      const matcher = matchers[matcherName];
+      this[matcherName] = applyMatcher.bind(null, matcherName, matcher, false, value);
+      this.not[matcherName] = applyMatcher.bind(null, matcherName, matcher, true, value);
+    }
+
+    function applyMatcher(matcherName, matcher, inverse, value, ...args) {
+      const result = matcher.call(null, value, ...args);
+      const message = result.message || `expect.${matcherName} failed`;
+      console.assert(result.pass !== inverse, message);
+    }
+  }
+}
+
+const DefaultMatchers = {
+  toBe: function(value, other, message) {
+    return { pass: value === other, message };
+  },
+
+  toBeFalsy: function(value, message) {
+    return { pass: !value, message };
+  },
+
+  toBeTruthy: function(value, message) {
+    return { pass: !!value, message };
+  },
+
+  toBeGreaterThan: function(value, other, message) {
+    return { pass: value > other, message };
+  },
+
+  toBeGreaterThanOrEqual: function(value, other, message) {
+    return { pass: value >= other, message };
+  },
+
+  toBeLessThan: function(value, other, message) {
+    return { pass: value < other, message };
+  },
+
+  toBeLessThanOrEqual: function(value, other, message) {
+    return { pass: value <= other, message };
+  },
+
+  toBeNull: function(value, message) {
+    return { pass: value === null, message };
+  },
+
+  toContain: function(value, other, message) {
+    return { pass: value.includes(other), message };
+  },
+
+  toEqual: function(value, other, message) {
+    return { pass: JSON.stringify(value) === JSON.stringify(other), message };
+  },
+
+  toBeCloseTo: function(value, other, precision, message) {
+    return {
+      pass: Math.abs(value - other) < Math.pow(10, -precision),
+      message
+    };
+  }
+};

utils/testrunner/Multimap.js

@@ -0,0 +1,95 @@
+/**
+ * Copyright 2017 Google Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+class Multimap {
+  constructor() {
+    this._map = new Map();
+  }
+
+  set(key, value) {
+    let set = this._map.get(key);
+    if (!set) {
+      set = new Set();
+      this._map.set(key, set);
+    }
+    set.add(value);
+  }
+
+  get(key) {
+    let result = this._map.get(key);
+    if (!result)
+      result = new Set();
+    return result;
+  }
+
+  has(key) {
+    return this._map.has(key);
+  }
+
+  hasValue(key, value) {
+    const set = this._map.get(key);
+    if (!set)
+      return false;
+    return set.has(value);
+  }
+
+  /**
+   * @return {number}
+   */
+  get size() {
+    return this._map.size;
+  }
+
+  delete(key, value) {
+    const values = this.get(key);
+    const result = values.delete(value);
+    if (!values.size)
+      this._map.delete(key);
+    return result;
+  }
+
+  deleteAll(key) {
+    this._map.delete(key);
+  }
+
+  firstValue(key) {
+    const set = this._map.get(key);
+    if (!set)
+      return null;
+    return set.values().next().value;
+  }
+
+  firstKey() {
+    return this._map.keys().next().value;
+  }
+
+  valuesArray() {
+    const result = [];
+    for (const key of this._map.keys())
+      result.push(...Array.from(this._map.get(key).values()));
+    return result;
+  }
+
+  keysArray() {
+    return Array.from(this._map.keys());
+  }
+
+  clear() {
+    this._map.clear();
+  }
+}
+
+module.exports = Multimap;

utils/testrunner/README.md

@@ -0,0 +1,45 @@
+# TestRunner
+
+- no additional binary required; tests are `node.js` scripts
+- parallel wrt IO operations
+- supports async/await
+- modular
+- well-isolated state per execution thread
+
+Example
+
+```js
+const {TestRunner, Reporter, Matchers} = require('../utils/testrunner');
+
+// Runner holds and runs all the tests
+const runner = new TestRunner({
+  parallel: 2, // run 2 parallel threads
+  timeout: 1000, // setup timeout of 1 second per test
+});
+// Simple expect-like matchers
+const {expect} = new Matchers();
+
+// Extract jasmine-like DSL into the global namespace
+const {describe, xdescribe, fdescribe} = runner;
+const {it, fit, xit} = runner;
+const {beforeAll, beforeEach, afterAll, afterEach} = runner;
+
+beforeAll(state => {
+  state.parallel; // this will be set with the execution thread id, either 0 or 1 in this example
+  state.foo = 'bar'; // set state for every test
+});
+
+describe('math', () => {
+  it('to be sane', async (state, test) => {
+    state.parallel; // Very first test will always be ran by the 0's thread
+    state.foo; // this will be 'bar'
+    expect(2 + 2).toBe(4);
+  });
+});
+
+// Reporter subscribes to TestRunner events and displays information in terminal
+const reporter = new Reporter(runner);
+
+// Run all tests.
+runner.run();
+```

utils/testrunner/Reporter.js

@@ -0,0 +1,119 @@
+/**
+ * Copyright 2017 Google Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const RED_COLOR = '\x1b[31m';
+const GREEN_COLOR = '\x1b[32m';
+const YELLOW_COLOR = '\x1b[33m';
+const RESET_COLOR = '\x1b[0m';
+
+class Reporter {
+  constructor(runner) {
+    this._runner = runner;
+    runner.on('started', this._onStarted.bind(this));
+    runner.on('terminated', this._onTerminated.bind(this));
+    runner.on('finished', this._onFinished.bind(this));
+    runner.on('teststarted', this._onTestStarted.bind(this));
+    runner.on('testfinished', this._onTestFinished.bind(this));
+  }
+
+  _onStarted() {
+    this._timestamp = Date.now();
+    console.log(`Running ${YELLOW_COLOR}${this._runner.parallel()}${RESET_COLOR} worker(s):\n`);
+  }
+
+  _onTerminated(message, error) {
+    this._printTestResults();
+    console.log(`${RED_COLOR}## TERMINATED ##${RESET_COLOR}`);
+    console.log('Message:');
+    console.log(`  ${RED_COLOR}${message}${RESET_COLOR}`);
+    if (error && error.stack) {
+      console.log('Stack:');
+      console.log(error.stack.split('\n').map(line => '  ' + line).join('\n'));
+    }
+    process.exit(2);
+  }
+
+  _onFinished() {
+    this._printTestResults();
+    const failedTests = this._runner.failedTests();
+    process.exit(failedTests.length > 0 ? 1 : 0);
+  }
+
+  _printTestResults() {
+    // 2 newlines after completing all tests.
+    console.log('\n');
+
+    const failedTests = this._runner.failedTests();
+    if (failedTests.length > 0) {
+      console.log('\nFailures:');
+      for (let i = 0; i < failedTests.length; ++i) {
+        const test = failedTests[i];
+        console.log(`${i + 1}) ${test.fullName}`);
+        if (test.result === 'timedout') {
+          console.log('  Message:');
+          console.log(`    ${YELLOW_COLOR}Timeout Exceeded ${this._runner.timeout()}ms${RESET_COLOR} ${formatLocation(test)}`);
+        } else {
+          console.log('  Message:');
+          console.log(`    ${RED_COLOR}${test.error.message || test.error}${RESET_COLOR} ${formatLocation(test)}`);
+          console.log('  Stack:');
+          if (test.error.stack)
+            console.log(test.error.stack.split('\n').map(line => '    ' + line).join('\n'));
+        }
+        console.log('');
+      }
+    }
+
+    const tests = this._runner.tests();
+    const skippedTests = tests.filter(test => test.result === 'skipped');
+    if (skippedTests.length > 0) {
+      console.log('\nSkipped:');
+      for (let i = 0; i < skippedTests.length; ++i) {
+        const test = skippedTests[i];
+        console.log(`${i + 1}) ${test.fullName}`);
+        console.log(`  ${YELLOW_COLOR}Temporary disabled with xit${RESET_COLOR} ${formatLocation(test)}\n`);
+      }
+    }
+
+    const executedTests = tests.filter(test => test.result);
+    console.log(`\nRan ${executedTests.length} of ${tests.length} test(s)`);
+    const milliseconds = Date.now() - this._timestamp;
+    const seconds = milliseconds / 1000;
+    console.log(`Finished in ${YELLOW_COLOR}${seconds}${RESET_COLOR} seconds`);
+
+    function formatLocation(test) {
+      const location = test.location;
+      if (!location)
+        return '';
+      return `@ ${location.fileName}:${location.lineNumber}:${location.columnNumber}`;
+    }
+  }
+
+  _onTestStarted() {
+  }
+
+  _onTestFinished(test) {
+    if (test.result === 'ok')
+      process.stdout.write(`${GREEN_COLOR}.${RESET_COLOR}`);
+    else if (test.result === 'skipped')
+      process.stdout.write(`${YELLOW_COLOR}*${RESET_COLOR}`);
+    else if (test.result === 'failed')
+      process.stdout.write(`${RED_COLOR}F${RESET_COLOR}`);
+    else if (test.result === 'timedout')
+      process.stdout.write(`${RED_COLOR}T${RESET_COLOR}`);
+  }
+}
+
+module.exports = Reporter;

utils/testrunner/TestRunner.js

@@ -0,0 +1,381 @@
+/**
+ * Copyright 2017 Google Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const path = require('path');
+const EventEmitter = require('events');
+const Multimap = require('./Multimap');
+
+const TimeoutError = new Error('Timeout');
+const TerminatedError = new Error('Terminated');
+
+class UserCallback {
+  constructor(callback, timeout) {
+    this._callback = callback;
+    this._terminatePromise = new Promise(resolve => {
+      this._terminateCallback = resolve;
+    });
+
+    this.timeout = timeout;
+    this.location = this._getLocation();
+  }
+
+  async run(...args) {
+    const timeoutPromise = new Promise(resolve => {
+      if (!this.timeout)
+        return;
+      setTimeout(resolve.bind(null, TimeoutError), this.timeout);
+    });
+    try {
+      return await Promise.race([
+        Promise.resolve().then(this._callback.bind(null, ...args)).then(() => null).catch(e => e),
+        timeoutPromise,
+        this._terminatePromise
+      ]);
+    } catch (e) {
+      return e;
+    }
+  }
+
+  _getLocation() {
+    const error = new Error();
+    const stackFrames = error.stack.split('\n').slice(1);
+    // Find first stackframe that doesn't point to this file.
+    for (let frame of stackFrames) {
+      frame = frame.trim();
+      if (!frame.startsWith('at '))
+        return null;
+      if (frame.endsWith(')')) {
+        const from = frame.indexOf('(');
+        frame = frame.substring(from + 1, frame.length - 1);
+      } else {
+        frame = frame.substring('at '.length + 1);
+      }
+
+      const match = frame.match(/^(.*):(\d+):(\d+)$/);
+      if (!match)
+        return null;
+      const filePath = match[1];
+      const lineNumber = match[2];
+      const columnNumber = match[3];
+      if (filePath === __filename)
+        continue;
+      const fileName = filePath.split(path.sep).pop();
+      return { fileName, filePath, lineNumber, columnNumber };
+    }
+    return null;
+  }
+
+  terminate() {
+    this._terminateCallback(TerminatedError);
+  }
+}
+
+const TestMode = {
+  Run: 'run',
+  Skip: 'skip',
+  Focus: 'focus'
+};
+
+const TestResult = {
+  Ok: 'ok',
+  Skipped: 'skipped', // User skipped the test
+  Failed: 'failed', // Exception happened during running
+  TimedOut: 'timedout', // Timeout Exceeded while running
+};
+
+class Test {
+  constructor(suite, name, callback, declaredMode, timeout) {
+    this.suite = suite;
+    this.name = name;
+    this.fullName = (suite.fullName + ' ' + name).trim();
+    this.declaredMode = declaredMode;
+    this._userCallback = new UserCallback(callback, timeout);
+    this.location = this._userCallback.location;
+
+    // Test results
+    this.result = null;
+    this.error = null;
+  }
+}
+
+class Suite {
+  constructor(parentSuite, name, declaredMode) {
+    this.parentSuite = parentSuite;
+    this.name = name;
+    this.fullName = (parentSuite ? parentSuite.fullName + ' ' + name : name).trim();
+    this.declaredMode = declaredMode;
+    /** @type {!Array<(!Test|!Suite)>} */
+    this.children = [];
+
+    this.beforeAll = null;
+    this.beforeEach = null;
+    this.afterAll = null;
+    this.afterEach = null;
+  }
+}
+
+class TestPass {
+  constructor(runner, rootSuite, tests, parallel) {
+    this._runner = runner;
+    this._parallel = parallel;
+    this._runningUserCallbacks = new Multimap();
+
+    this._rootSuite = rootSuite;
+    this._workerDistribution = new Multimap();
+
+    let workerId = 0;
+    for (const test of tests) {
+      // Reset results for tests that will be run.
+      test.result = null;
+      test.error = null;
+      this._workerDistribution.set(test, workerId);
+      for (let suite = test.suite; suite; suite = suite.parentSuite)
+        this._workerDistribution.set(suite, workerId);
+      // Do not shard skipped tests across workers.
+      if (test.declaredMode !== TestMode.Skip)
+        workerId = (workerId + 1) % parallel;
+    }
+
+    this._termination = null;
+  }
+
+  async run() {
+    const terminations = [
+      createTermination.call(this, 'SIGINT', 'SIGINT received'),
+      createTermination.call(this, 'SIGHUP', 'SIGHUP received'),
+      createTermination.call(this, 'SIGTERM', 'SIGTERM received'),
+      createTermination.call(this, 'unhandledRejection', 'UNHANDLED PROMISE REJECTION'),
+    ];
+    for (const termination of terminations)
+      process.on(termination.event, termination.handler);
+
+    const workerPromises = [];
+    for (let i = 0; i < this._parallel; ++i)
+      workerPromises.push(this._runSuite(i, [this._rootSuite], {parallel: i}));
+    await Promise.all(workerPromises);
+
+    for (const termination of terminations)
+      process.removeListener(termination.event, termination.handler);
+    return this._termination;
+
+    function createTermination(event, message) {
+      return {
+        event,
+        message,
+        handler: error => this._terminate(message, error)
+      };
+    }
+  }
+
+  async _runSuite(workerId, suitesStack, state) {
+    if (this._termination)
+      return;
+    const currentSuite = suitesStack[suitesStack.length - 1];
+    if (!this._workerDistribution.hasValue(currentSuite, workerId))
+      return;
+    await this._runHook(workerId, currentSuite, 'beforeAll', state);
+    for (const child of currentSuite.children) {
+      if (!this._workerDistribution.hasValue(child, workerId))
+        continue;
+      if (child instanceof Test) {
+        for (let i = 0; i < suitesStack.length; i++)
+          await this._runHook(workerId, suitesStack[i], 'beforeEach', state, child);
+        await this._runTest(workerId, child, state);
+        for (let i = suitesStack.length - 1; i >= 0; i--)
+          await this._runHook(workerId, suitesStack[i], 'afterEach', state, child);
+      } else {
+        suitesStack.push(child);
+        await this._runSuite(workerId, suitesStack, state);
+        suitesStack.pop();
+      }
+    }
+    await this._runHook(workerId, currentSuite, 'afterAll', state);
+  }
+
+  async _runTest(workerId, test, state) {
+    if (this._termination)
+      return;
+    this._runner._willStartTest(test);
+    if (test.declaredMode === TestMode.Skip) {
+      test.result = TestResult.Skipped;
+      this._runner._didFinishTest(test);
+      return;
+    }
+    this._runningUserCallbacks.set(workerId, test._userCallback);
+    const error = await test._userCallback.run(state, test);
+    this._runningUserCallbacks.delete(workerId, test._userCallback);
+    if (this._termination)
+      return;
+    test.error = error;
+    if (!error)
+      test.result = TestResult.Ok;
+    else if (test.error === TimeoutError)
+      test.result = TestResult.TimedOut;
+    else
+      test.result = TestResult.Failed;
+    this._runner._didFinishTest(test);
+  }
+
+  async _runHook(workerId, suite, hookName, ...args) {
+    if (this._termination)
+      return;
+    const hook = suite[hookName];
+    if (!hook)
+      return;
+    this._runningUserCallbacks.set(workerId, hook);
+    const error = await hook.run(...args);
+    this._runningUserCallbacks.delete(workerId, hook);
+    if (error === TimeoutError) {
+      const location = `${hook.location.fileName}:${hook.location.lineNumber}:${hook.location.columnNumber}`;
+      const message = `${location} - Timeout Exceeded ${hook.timeout}ms while running "${hookName}" in suite "${suite.fullName}"`;
+      this._terminate(message, null);
+    } else if (error) {
+      const location = `${hook.location.fileName}:${hook.location.lineNumber}:${hook.location.columnNumber}`;
+      const message = `${location} - FAILED while running "${hookName}" in suite "${suite.fullName}"`;
+      this._terminate(message, error);
+    }
+  }
+
+  _terminate(message, error) {
+    if (this._termination)
+      return;
+    this._termination = {message, error};
+    for (const userCallback of this._runningUserCallbacks.valuesArray())
+      userCallback.terminate();
+  }
+}
+
+class TestRunner extends EventEmitter {
+  constructor(options = {}) {
+    super();
+    this._rootSuite = new Suite(null, '', TestMode.Run);
+    this._currentSuite = this._rootSuite;
+    this._tests = [];
+    this._timeout = options.timeout || 10 * 1000; // 10 seconds.
+    this._parallel = options.parallel || 1;
+    this._retryFailures = !!options.retryFailures;
+
+    this._hasFocusedTestsOrSuites = false;
+
+    // bind methods so that they can be used as a DSL.
+    this.describe = this._addSuite.bind(this, TestMode.Run);
+    this.fdescribe = this._addSuite.bind(this, TestMode.Focus);
+    this.xdescribe = this._addSuite.bind(this, TestMode.Skip);
+    this.it = this._addTest.bind(this, TestMode.Run);
+    this.fit = this._addTest.bind(this, TestMode.Focus);
+    this.xit = this._addTest.bind(this, TestMode.Skip);
+    this.beforeAll = this._addHook.bind(this, 'beforeAll');
+    this.beforeEach = this._addHook.bind(this, 'beforeEach');
+    this.afterAll = this._addHook.bind(this, 'afterAll');
+    this.afterEach = this._addHook.bind(this, 'afterEach');
+  }
+
+  _addTest(mode, name, callback) {
+    const test = new Test(this._currentSuite, name, callback, mode, this._timeout);
+    this._currentSuite.children.push(test);
+    this._tests.push(test);
+    this._hasFocusedTestsOrSuites = this._hasFocusedTestsOrSuites || mode === TestMode.Focus;
+  }
+
+  _addSuite(mode, name, callback) {
+    const oldSuite = this._currentSuite;
+    const suite = new Suite(this._currentSuite, name, mode);
+    this._currentSuite.children.push(suite);
+    this._currentSuite = suite;
+    callback();
+    this._currentSuite = oldSuite;
+    this._hasFocusedTestsOrSuites = this._hasFocusedTestsOrSuites || mode === TestMode.Focus;
+  }
+
+  _addHook(hookName, callback) {
+    console.assert(this._currentSuite[hookName] === null, `Only one ${hookName} hook available per suite`);
+    const hook = new UserCallback(callback, this._timeout);
+    this._currentSuite[hookName] = hook;
+  }
+
+  async run() {
+    this.emit(TestRunner.Events.Started);
+    const pass = new TestPass(this, this._rootSuite, this._runnableTests(), this._parallel);
+    const termination = await pass.run();
+    if (termination)
+      this.emit(TestRunner.Events.Terminated, termination.message, termination.error);
+    else
+      this.emit(TestRunner.Events.Finished);
+  }
+
+  timeout() {
+    return this._timeout;
+  }
+
+  _runnableTests() {
+    if (!this._hasFocusedTestsOrSuites)
+      return this._tests;
+
+    const tests = [];
+    const blacklistSuites = new Set();
+    // First pass: pick "fit" and blacklist parent suites
+    for (const test of this._tests) {
+      if (test.declaredMode !== TestMode.Focus)
+        continue;
+      tests.push(test);
+      for (let suite = test.suite; suite; suite = suite.parentSuite)
+        blacklistSuites.add(suite);
+    }
+    // Second pass: pick all tests that belong to non-blacklisted "fdescribe"
+    for (const test of this._tests) {
+      let insideFocusedSuite = false;
+      for (let suite = test.suite; suite; suite = suite.parentSuite) {
+        if (!blacklistSuites.has(suite) && suite.declaredMode === TestMode.Focus) {
+          insideFocusedSuite = true;
+          break;
+        }
+      }
+      if (insideFocusedSuite)
+        tests.push(test);
+    }
+    return tests;
+  }
+
+  tests() {
+    return this._tests.slice();
+  }
+
+  failedTests() {
+    return this._tests.filter(test => test.result === 'failed' || test.result === 'timedout');
+  }
+
+  parallel() {
+    return this._parallel;
+  }
+
+  _willStartTest(test) {
+    this.emit('teststarted', test);
+  }
+
+  _didFinishTest(test) {
+    this.emit('testfinished', test);
+  }
+}
+
+TestRunner.Events = {
+  Started: 'started',
+  TestStarted: 'teststarted',
+  TestFinished: 'testfinished',
+  Terminated: 'terminated',
+  Finished: 'finished',
+};
+
+module.exports = TestRunner;

utils/testrunner/examples/fail.js

@@ -0,0 +1,33 @@
+/**
+ * Copyright 2017 Google Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const {TestRunner, Reporter, Matchers} = require('..');
+
+const runner = new TestRunner();
+const reporter = new Reporter(runner);
+const {expect} = new Matchers();
+
+const {describe, xdescribe, fdescribe} = runner;
+const {it, fit, xit} = runner;
+const {beforeAll, beforeEach, afterAll, afterEach} = runner;
+
+describe('testsuite', () => {
+  it('failure', async (state) => {
+    expect(false).toBeTruthy();
+  });
+});
+
+runner.run();

utils/testrunner/examples/hookfail.js

@@ -0,0 +1,35 @@
+/**
+ * Copyright 2017 Google Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const {TestRunner, Reporter, Matchers} = require('..');
+
+const runner = new TestRunner();
+const reporter = new Reporter(runner);
+const {expect} = new Matchers();
+
+const {describe, xdescribe, fdescribe} = runner;
+const {it, fit, xit} = runner;
+const {beforeAll, beforeEach, afterAll, afterEach} = runner;
+
+describe('testsuite', () => {
+  beforeAll(() => {
+    expect(false).toBeTruthy();
+  });
+  it('test', async () => {
+  });
+});
+
+runner.run();

utils/testrunner/examples/hooktimeout.js

@@ -0,0 +1,35 @@
+/**
+ * Copyright 2017 Google Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const {TestRunner, Reporter, Matchers} = require('..');
+
+const runner = new TestRunner({ timeout: 100 });
+const reporter = new Reporter(runner);
+const {expect} = new Matchers();
+
+const {describe, xdescribe, fdescribe} = runner;
+const {it, fit, xit} = runner;
+const {beforeAll, beforeEach, afterAll, afterEach} = runner;
+
+describe('testsuite', () => {
+  beforeAll(async () => {
+    await new Promise(() => {});
+  });
+  it('something', async (state) => {
+  });
+});
+
+runner.run();

utils/testrunner/examples/timeout.js

@@ -0,0 +1,32 @@
+/**
+ * Copyright 2017 Google Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const {TestRunner, Reporter} = require('..');
+
+const runner = new TestRunner({ timeout: 100 });
+const reporter = new Reporter(runner);
+
+const {describe, xdescribe, fdescribe} = runner;
+const {it, fit, xit} = runner;
+const {beforeAll, beforeEach, afterAll, afterEach} = runner;
+
+describe('testsuite', () => {
+  it('timeout', async (state) => {
+    await new Promise(() => {});
+  });
+});
+
+runner.run();

utils/testrunner/examples/unhandledpromiserejection.js

@@ -0,0 +1,35 @@
+/**
+ * Copyright 2017 Google Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const {TestRunner, Reporter} = require('..');
+
+const runner = new TestRunner();
+const reporter = new Reporter(runner);
+
+const {describe, xdescribe, fdescribe} = runner;
+const {it, fit, xit} = runner;
+const {beforeAll, beforeEach, afterAll, afterEach} = runner;
+
+describe('testsuite', () => {
+  it('failure', async (state) => {
+    Promise.reject(new Error('fail!'));
+  });
+  it('slow', async () => {
+    await new Promise(x => setTimeout(x, 1000));
+  });
+});
+
+runner.run();

utils/testrunner/index.js

@@ -0,0 +1,21 @@
+/**
+ * Copyright 2017 Google Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const TestRunner = require('./TestRunner');
+const Reporter = require('./Reporter');
+const Matchers = require('./Matchers');
+
+module.exports = { TestRunner, Reporter, Matchers };