chore: introduce initial version of flakiness dashboard (#4781)

This patch introduces a dashboard that records test results and
uploads them to https://github.com/aslushnikov/puppeteer-flakiness-dashboard

Since many bots might push results in parallel, each bot pushes
results to its own git branch.

FlakinessDashboard also generates a simple README.md with a flakiness
summary. If this proves to be not enough, we can build a website that
fetches flakiness data and renders it nicely.

.cirrus.yml

@@ -1,5 +1,9 @@
 env:
   DISPLAY: :99.0
+  FLAKINESS_DASHBOARD_PASSWORD: ENCRYPTED[b3e207db5d153b543f219d3c3b9123d8321834b783b9e45ac7d380e026ab3a56398bde51b521ac5859e7e45cb95d0992]
+  FLAKINESS_DASHBOARD_NAME: Cirrus ${CIRRUS_TASK_NAME}
+  FLAKINESS_DASHBOARD_BUILD_SHA: ${CIRRUS_CHANGE_IN_REPO}
+  FLAKINESS_DASHBOARD_BUILD_URL: https://cirrus-ci.com/build/${CIRRUS_BUILD_ID}
 
 task:
   matrix:

test/test.js

@@ -111,4 +111,7 @@ new Reporter(testRunner, {
   projectFolder: utils.projectRoot(),
   showSlowTests: process.env.CI ? 5 : 0,
 });
+
+utils.initializeFlakinessDashboardIfNeeded(testRunner);
 testRunner.run();
+

test/utils.js

@@ -16,6 +16,7 @@
 
 const fs = require('fs');
 const path = require('path');
+const {FlakinessDashboard} = require('../utils/flakiness-dashboard');
 const PROJECT_ROOT = fs.existsSync(path.join(__dirname, '..', 'package.json')) ? path.join(__dirname, '..') : path.join(__dirname, '..', '..');
 
 /**
@@ -153,4 +154,70 @@ const utils = module.exports = {
       });
     });
   },
+
+  initializeFlakinessDashboardIfNeeded: function(testRunner) {
+    // Generate testIDs for all tests and verify they don't clash.
+    // This will add |test.testId| for every test.
+    //
+    // NOTE: we do this unconditionally so that developers can see problems in
+    // their local setups.
+    generateTestIDs(testRunner);
+    // FLAKINESS_DASHBOARD_PASSWORD is an encrypted/secured variable.
+    // Encrypted variables get a special treatment in CI's when handling PRs so that
+    // secrets are not leaked to untrusted code.
+    // - AppVeyor DOES NOT decrypt secured variables for PRs
+    // - Travis DOES NOT decrypt encrypted variables for PRs
+    // - Cirrus CI DOES NOT decrypt encrypted variables for PRs *unless* PR is sent
+    //   from someone who has WRITE ACCESS to the repo.
+    //
+    // Since we don't want to run flakiness dashboard for PRs on all CIs, we
+    // check existance of FLAKINESS_DASHBOARD_PASSWORD and absense of
+    // CIRRUS_BASE_SHA env variables.
+    if (!process.env.FLAKINESS_DASHBOARD_PASSWORD || process.env.CIRRUS_BASE_SHA)
+      return;
+    const sha = process.env.FLAKINESS_DASHBOARD_BUILD_SHA;
+    const dashboard = new FlakinessDashboard({
+      dashboardName: process.env.FLAKINESS_DASHBOARD_NAME,
+      build: {
+        url: process.env.FLAKINESS_DASHBOARD_BUILD_URL,
+        name: sha.substring(0, 8),
+      },
+      dashboardRepo: {
+        url: 'https://github.com/aslushnikov/puppeteer-flakiness-dashboard.git',
+        username: 'puppeteer-flakiness',
+        email: 'aslushnikov+puppeteerflakiness@gmail.com',
+        password: process.env.FLAKINESS_DASHBOARD_PASSWORD,
+      },
+    });
+
+    testRunner.on('testfinished', test => {
+      const testpath = test.location.filePath.substring(utils.projectRoot().length);
+      const url = `https://github.com/GoogleChrome/puppeteer/blob/${sha}/${testpath}#L${test.location.lineNumber}`;
+      dashboard.reportTestResult({
+        testId: test.testId,
+        name: test.location.fileName + ':' + test.location.lineNumber,
+        description: test.fullName,
+        url,
+        result: test.result,
+      });
+    });
+    testRunner.on('terminated', () => dashboard.uploadAndCleanup());
+    testRunner.on('finished', () => dashboard.uploadAndCleanup());
+
+    function generateTestIDs(testRunner) {
+      const testIds = new Map();
+      for (const test of testRunner.tests()) {
+        const testIdComponents = [test.name];
+        for (let suite = test.suite; !!suite.parentSuite; suite = suite.parentSuite)
+          testIdComponents.push(suite.name);
+        testIdComponents.reverse();
+        const testId = testIdComponents.join('>');
+        const clashingTest = testIds.get(testId);
+        if (clashingTest)
+          throw new Error(`Two tests with clashing IDs: ${test.location.fileName}:${test.location.lineNumber} and ${clashingTest.location.fileName}:${clashingTest.location.lineNumber}`);
+        testIds.set(testId, test);
+        test.testId = testId;
+      }
+    }
+  },
 };

utils/flakiness-dashboard/FlakinessDashboard.js

@@ -0,0 +1,270 @@
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const spawn = require('child_process').spawn;
+const debug = require('debug')('flakiness');
+
+const rmAsync = promisify(require('rimraf'));
+const mkdtempAsync = promisify(fs.mkdtemp);
+const readFileAsync = promisify(fs.readFile);
+const writeFileAsync = promisify(fs.writeFile);
+
+const TMP_FOLDER = path.join(os.tmpdir(), 'flakiness_tmp_folder-');
+
+const RED_COLOR = '\x1b[31m';
+const GREEN_COLOR = '\x1b[32m';
+const YELLOW_COLOR = '\x1b[33m';
+const RESET_COLOR = '\x1b[0m';
+
+class FlakinessDashboard {
+  constructor({dashboardName, build, dashboardRepo, options}) {
+    this._dashboardName = dashboardName;
+    this._dashboardRepo = dashboardRepo;
+    this._options = options;
+    this._build = new Build(Date.now(), build.name, build.url, []);
+  }
+
+  reportTestResult(test) {
+    this._build.reportTestResult(test);
+  }
+
+  async uploadAndCleanup() {
+    console.log(`\n${YELLOW_COLOR}=== UPLOADING Flakiness Dashboard${RESET_COLOR}`);
+    const startTimestamp = Date.now();
+    const branch = this._dashboardRepo.branch || this._dashboardName.trim().toLowerCase().replace(/\s/g, '-').replace(/[^-0-9a-zа-яё]/ig, '');
+    const git = await Git.initialize(this._dashboardRepo.url, branch, this._dashboardRepo.username, this._dashboardRepo.email, this._dashboardRepo.password);
+    console.log(`  > Dashboard Location: ${git.path()}`);
+
+    // Do at max 5 attempts to upload changes to github.
+    let success = false;
+    const MAX_ATTEMPTS = 7;
+    for (let i = 0; !success && i < MAX_ATTEMPTS; ++i) {
+      const dashboard = await Dashboard.create(this._dashboardName, git.path(), this._options);
+      dashboard.addBuild(this._build);
+      await dashboard.saveJSON();
+      await dashboard.generateReadme();
+      // if push went through - great! We're done!
+      if (await git.commitAllAndPush()) {
+        success = true;
+        console.log(`  > Push attempt ${YELLOW_COLOR}${i + 1}${RESET_COLOR} of ${YELLOW_COLOR}${MAX_ATTEMPTS}${RESET_COLOR}: ${GREEN_COLOR}SUCCESS${RESET_COLOR}`);
+      } else {
+        // Otherwise - wait random time between 3 and 11 seconds.
+        const cooldown = 3000 + Math.round(Math.random() * 1000) * 8;
+        console.log(`  > Push attempt ${YELLOW_COLOR}${i + 1}${RESET_COLOR} of ${YELLOW_COLOR}${MAX_ATTEMPTS}${RESET_COLOR}: ${RED_COLOR}FAILED${RESET_COLOR}, cooldown ${YELLOW_COLOR}${cooldown / 1000}${RESET_COLOR} seconds`);
+        await new Promise(x => setTimeout(x, cooldown));
+        // Reset our generated dashboard and pull from origin.
+        await git.hardResetToOriginMaster();
+        await git.pullFromOrigin();
+      }
+    }
+    await rmAsync(git.path());
+    console.log(`  > TOTAL TIME: ${YELLOW_COLOR}${(Date.now() - startTimestamp) / 1000}${RESET_COLOR} seconds`);
+    if (success)
+      console.log(`${YELLOW_COLOR}=== COMPLETE${RESET_COLOR}`);
+    else
+      console.log(`${RED_COLOR}=== FAILED${RESET_COLOR}`);
+    console.log('');
+  }
+}
+
+const DASHBOARD_VERSION = 1;
+const DASHBOARD_FILENAME = 'dashboard.json';
+
+class Dashboard {
+  static async create(name, dashboardPath, options = {}) {
+    const filePath = path.join(dashboardPath, DASHBOARD_FILENAME);
+    let data = null;
+    try {
+      data = JSON.parse(await readFileAsync(filePath));
+    } catch (e) {
+      // Looks like there's no dashboard yet - create one.
+      return new Dashboard(name, dashboardPath, [], options);
+    }
+    if (!data.version)
+      throw new Error('cannot parse dashboard data: missing "version" field!');
+    if (data.version > DASHBOARD_VERSION)
+      throw new Error('cannot manage dashboards that are newer then this');
+    const builds = data.builds.map(build => new Build(build.timestamp, build.name, build.url, build.tests));
+    return new Dashboard(name, dashboardPath, builds, options);
+  }
+
+  async saveJSON() {
+    const data = { version: DASHBOARD_VERSION };
+    data.builds = this._builds.map(build => ({
+      timestamp: build._timestamp,
+      name: build._name,
+      url: build._url,
+      tests: build._tests,
+    }));
+    await writeFileAsync(path.join(this._dashboardPath, DASHBOARD_FILENAME), JSON.stringify(data, null, 2));
+  }
+
+  async generateReadme() {
+    const flakyTests = new Map();
+    for (const build of this._builds) {
+      for (const test of build._tests) {
+        if (test.result !== 'ok')
+          flakyTests.set(test.testId, test);
+      }
+    }
+
+    const text = [];
+    text.push(`# ${this._name}`);
+    text.push(``);
+
+    for (const [testId, test] of flakyTests) {
+      text.push(`#### [${test.name}](${test.url}) - ${test.description}`);
+      text.push('');
+
+      let headers = '|';
+      let splitters = '|';
+      let dataColumns = '|';
+      for (let i = this._builds.length - 1; i >= 0; --i) {
+        const build = this._builds[i];
+        headers += ` [${build._name}](${build._url}) |`;
+        splitters += ' :---: |';
+        const test = build._testsMap.get(testId);
+        if (test) {
+          const r = test.result.toLowerCase();
+          let text = r;
+          if (r === 'ok')
+            text = '✅';
+          else if (r.includes('fail'))
+            text = '🛑';
+          dataColumns += ` [${text}](${test.url}) |`;
+        } else {
+          dataColumns += ` missing |`;
+        }
+      }
+      text.push(headers);
+      text.push(splitters);
+      text.push(dataColumns);
+      text.push('');
+    }
+
+    await writeFileAsync(path.join(this._dashboardPath, 'README.md'), text.join('\n'));
+  }
+
+  constructor(name, dashboardPath, builds, options) {
+    const {
+      maxBuilds = 30,
+    } = options;
+    this._name = name;
+    this._dashboardPath = dashboardPath;
+    this._builds = builds.slice(builds.length - maxBuilds);
+  }
+
+  addBuild(build) {
+    this._builds.push(build);
+  }
+}
+
+class Build {
+  constructor(timestamp, name, url, tests) {
+    this._timestamp = timestamp;
+    this._name = name;
+    this._url = url;
+    this._tests = tests;
+    this._testsMap = new Map();
+    for (const test of tests)
+      this._testsMap.set(test.testId, test);
+  }
+
+  reportTestResult(test) {
+    this._tests.push(test);
+    this._testsMap.set(test.testId, test);
+  }
+}
+
+module.exports = {FlakinessDashboard};
+
+function promisify(nodeFunction) {
+  function promisified(...args) {
+    return new Promise((resolve, reject) => {
+      function callback(err, ...result) {
+        if (err)
+          return reject(err);
+        if (result.length === 1)
+          return resolve(result[0]);
+        return resolve(result);
+      }
+      nodeFunction.call(null, ...args, callback);
+    });
+  }
+  return promisified;
+}
+
+class Git {
+  static async initialize(url, branch, username, email, password) {
+    let schemeIndex = url.indexOf('://');
+    if (schemeIndex === -1)
+      throw new Error(`Malformed URL "${url}": expected to start with "https://"`);
+    schemeIndex += '://'.length;
+    url = url.substring(0, schemeIndex) + username + ':' + password + '@' + url.substring(schemeIndex);
+    const repoPath = await mkdtempAsync(TMP_FOLDER);
+    // Check existance of a remote branch for this bot.
+    const {stdout} = await spawnAsync('git', 'ls-remote', '--heads', url, branch);
+    // If there is no remote branch for this bot - create one.
+    if (!stdout.includes(branch)) {
+      await spawnAsyncOrDie('git', 'clone', '--no-checkout', '--depth=1', url, repoPath);
+
+      await spawnAsyncOrDie('git', 'checkout', '--orphan', branch, {cwd: repoPath});
+      await spawnAsyncOrDie('git', 'reset', '--hard', {cwd: repoPath});
+    } else {
+      await spawnAsyncOrDie('git', 'clone', '--single-branch', '--branch', `${branch}`, '--depth=1', url, repoPath);
+    }
+    await spawnAsyncOrDie('git', 'config', 'user.email', `"${email}"`, {cwd: repoPath});
+    await spawnAsyncOrDie('git', 'config', 'user.name', `"${username}"`, {cwd: repoPath});
+    return new Git(repoPath, url, branch, username);
+  }
+
+  async commitAllAndPush() {
+    await spawnAsyncOrDie('git', 'add', '.', {cwd: this._repoPath});
+    await spawnAsyncOrDie('git', 'commit', '-m', '"update dashboard"', '--author', '"puppeteer-flakiness <aslushnikov+puppeteerflakiness@gmail.com>"', {cwd: this._repoPath});
+    const {code} = await spawnAsync('git', 'push', 'origin', this._branch, {cwd: this._repoPath});
+    return code === 0;
+  }
+
+  async hardResetToOriginMaster() {
+    await spawnAsyncOrDie('git', 'reset', '--hard', `origin/${this._branch}`, {cwd: this._repoPath});
+  }
+
+  async pullFromOrigin() {
+    await spawnAsyncOrDie('git', 'pull', 'origin', this._branch, {cwd: this._repoPath});
+  }
+
+  constructor(repoPath, url, branch, username) {
+    this._repoPath = repoPath;
+    this._url = url;
+    this._branch = branch;
+    this._username = username;
+  }
+
+  path() {
+    return this._repoPath;
+  }
+}
+
+async function spawnAsync(command, ...args) {
+  let options = {};
+  if (args.length && args[args.length - 1].constructor.name !== 'String')
+    options = args.pop();
+  const cmd = spawn(command, args, options);
+  let stdout = '';
+  let stderr = '';
+  cmd.stdout.on('data', data => stdout += data);
+  cmd.stderr.on('data', data => stderr += data);
+  const code = await new Promise(x => cmd.once('close', x));
+  if (stdout)
+    debug(stdout);
+  if (stderr)
+    debug(stderr);
+  return {code, stdout, stderr};
+}
+
+async function spawnAsyncOrDie(command, ...args) {
+  const {code, stdout, stderr} = await spawnAsync(command, ...args);
+  if (code !== 0)
+    throw new Error(`Failed to executed: "${command} ${args.join(' ')}".\n\n=== STDOUT ===\n${stdout}\n\n\n=== STDERR ===\n${stderr}`);
+  return {stdout, stderr};
+}

utils/flakiness-dashboard/index.js

@@ -0,0 +1,3 @@
+const {FlakinessDashboard} = require('./FlakinessDashboard');
+
+module.exports = {FlakinessDashboard};

utils/node6-transform/index.js

@@ -35,6 +35,7 @@ copyFolder(path.join(root, 'lib'), path.join(dest, 'lib'));
 copyFolder(path.join(root, 'test'), path.join(dest, 'test'));
 copyFolder(path.join(root, 'utils', 'testrunner'), path.join(dest, 'utils', 'testrunner'));
 copyFolder(path.join(root, 'utils', 'testserver'), path.join(dest, 'utils', 'testserver'));
+copyFolder(path.join(root, 'utils', 'flakiness-dashboard'), path.join(dest, 'utils', 'flakiness-dashboard'));
 
 function copyFolder(source, target) {
   if (fs.existsSync(target))