chore: remove flakiness dashboard (#5592)

We're not using it at all so might as well cut down on code in the codebase.

.appveyor.yml

@@ -1,14 +1,10 @@
 environment:
   matrix:
     - nodejs_version: "10.18.1"
-      FLAKINESS_DASHBOARD_NAME: Appveyor Chromium (Win + node10)
-  FLAKINESS_DASHBOARD_PASSWORD:
-    secure: g66jP+j6C+hkXLutBV9fdxB5fRJgcQQzy93SgQzXUmcCl/RjkJwnzyHvX0xfCVnv
 
 build: off
 
 install:
-  - ps: $env:FLAKINESS_DASHBOARD_BUILD_URL="https://ci.appveyor.com/project/aslushnikov/puppeteer/builds/$env:APPVEYOR_BUILD_ID/job/$env:APPVEYOR_JOB_ID"
   - ps: Install-Product node $env:nodejs_version
   - npm install
   - if "%nodejs_version%" == "10.18.1" (

.cirrus.yml

@@ -1,8 +1,5 @@
 env:
   DISPLAY: :99.0
-  FLAKINESS_DASHBOARD_PASSWORD: ENCRYPTED[b3e207db5d153b543f219d3c3b9123d8321834b783b9e45ac7d380e026ab3a56398bde51b521ac5859e7e45cb95d0992]
-  FLAKINESS_DASHBOARD_NAME: Cirrus ${CIRRUS_TASK_NAME}
-  FLAKINESS_DASHBOARD_BUILD_URL: https://cirrus-ci.com/task/${CIRRUS_TASK_ID}
 
 task:
   matrix:

.travis.yml

@@ -31,16 +31,10 @@ jobs:
     - node_js: "12.16.1"
       env:
         - CHROMIUM=true
-        - FLAKINESS_DASHBOARD_NAME="Travis Chromium (node12 + linux)"
-        - FLAKINESS_DASHBOARD_BUILD_URL="${TRAVIS_JOB_WEB_URL}"
     - node_js: "10.18.1"
       env:
         - CHROMIUM=true
-        - FLAKINESS_DASHBOARD_NAME="Travis Chromium (node10 + linux)"
-        - FLAKINESS_DASHBOARD_BUILD_URL="${TRAVIS_JOB_WEB_URL}"
     - node_js: "10.18.1"
       env:
         - FIREFOX=true
-        - FLAKINESS_DASHBOARD_NAME="Travis Firefox Nightly (node10 + linux)"
-        - FLAKINESS_DASHBOARD_BUILD_URL="${TRAVIS_JOB_WEB_URL}"
         - FIREFOX_HOME=$TRAVIS_HOME/firefox-latest

test/test.js

@@ -116,7 +116,6 @@ new Reporter(testRunner, {
 });
 
 (async() => {
-  await utils.initializeFlakinessDashboardIfNeeded(testRunner);
   testRunner.run();
 })();
 

test/utils.js

@@ -18,7 +18,6 @@ const fs = require('fs');
 const path = require('path');
 const expect = require('expect');
 const GoldenUtils = require('./golden-utils');
-const {FlakinessDashboard} = require('../utils/flakiness-dashboard');
 const PROJECT_ROOT = fs.existsSync(path.join(__dirname, '..', 'package.json')) ? path.join(__dirname, '..') : path.join(__dirname, '..', '..');
 
 const COVERAGE_TESTSUITE_NAME = '**API COVERAGE**';
@@ -177,80 +176,4 @@ const utils = module.exports = {
       });
     });
   },
-
-  initializeFlakinessDashboardIfNeeded: async function(testRunner) {
-    // Generate testIDs for all tests and verify they don't clash.
-    // This will add |test.testId| for every test.
-    //
-    // NOTE: we do this on CI's so that problems arise on PR trybots.
-    if (process.env.CI)
-      generateTestIDs(testRunner);
-    // FLAKINESS_DASHBOARD_PASSWORD is an encrypted/secured variable.
-    // Encrypted variables get a special treatment in CI's when handling PRs so that
-    // secrets are not leaked to untrusted code.
-    // - AppVeyor DOES NOT decrypt secured variables for PRs
-    // - Travis DOES NOT decrypt encrypted variables for PRs
-    // - Cirrus CI DOES NOT decrypt encrypted variables for PRs *unless* PR is sent
-    //   from someone who has WRITE ACCESS to the repo.
-    //
-    // Since we don't want to run flakiness dashboard for PRs on all CIs, we
-    // check existence of FLAKINESS_DASHBOARD_PASSWORD and absense of
-    // CIRRUS_BASE_SHA env variables.
-    if (!process.env.FLAKINESS_DASHBOARD_PASSWORD || process.env.CIRRUS_BASE_SHA)
-      return;
-    const {sha, timestamp} = await FlakinessDashboard.getCommitDetails(__dirname, 'HEAD');
-    const dashboard = new FlakinessDashboard({
-      commit: {
-        sha,
-        timestamp,
-        url: `https://github.com/puppeteer/puppeteer/commit/${sha}`,
-      },
-      build: {
-        url: process.env.FLAKINESS_DASHBOARD_BUILD_URL,
-      },
-      dashboardRepo: {
-        url: 'https://github.com/aslushnikov/puppeteer-flakiness-dashboard.git',
-        username: 'puppeteer-flakiness',
-        email: 'aslushnikov+puppeteerflakiness@gmail.com',
-        password: process.env.FLAKINESS_DASHBOARD_PASSWORD,
-        branch: process.env.FLAKINESS_DASHBOARD_NAME,
-      },
-    });
-
-    testRunner.on('testfinished', test => {
-      // Do not report tests from COVERAGE testsuite.
-      // They don't bring much value to us.
-      if (test.fullName.includes(COVERAGE_TESTSUITE_NAME))
-        return;
-      const testpath = test.location.filePath.substring(utils.projectRoot().length);
-      const url = `https://github.com/puppeteer/puppeteer/blob/${sha}/${testpath}#L${test.location.lineNumber}`;
-      dashboard.reportTestResult({
-        testId: test.testId,
-        name: test.location.fileName + ':' + test.location.lineNumber,
-        description: test.fullName,
-        url,
-        result: test.result,
-      });
-    });
-    testRunner.on('finished', async({result}) => {
-      dashboard.setBuildResult(result);
-      await dashboard.uploadAndCleanup();
-    });
-
-    function generateTestIDs(testRunner) {
-      const testIds = new Map();
-      for (const test of testRunner.tests()) {
-        const testIdComponents = [test.name];
-        for (let suite = test.suite; !!suite.parentSuite; suite = suite.parentSuite)
-          testIdComponents.push(suite.name);
-        testIdComponents.reverse();
-        const testId = testIdComponents.join('>');
-        const clashingTest = testIds.get(testId);
-        if (clashingTest)
-          throw new Error(`Two tests with clashing IDs: ${test.location.fileName}:${test.location.lineNumber} and ${clashingTest.location.fileName}:${clashingTest.location.lineNumber}`);
-        testIds.set(testId, test);
-        test.testId = testId;
-      }
-    }
-  },
 };

utils/flakiness-dashboard/FlakinessDashboard.js

@@ -1,218 +0,0 @@
-const fs = require('fs');
-const os = require('os');
-const path = require('path');
-const spawn = require('child_process').spawn;
-const debug = require('debug')('flakiness');
-
-const rmAsync = promisify(require('rimraf'));
-const mkdtempAsync = promisify(fs.mkdtemp);
-const readFileAsync = promisify(fs.readFile);
-const writeFileAsync = promisify(fs.writeFile);
-
-const TMP_FOLDER = path.join(os.tmpdir(), 'flakiness_tmp_folder-');
-
-const RED_COLOR = '\x1b[31m';
-const GREEN_COLOR = '\x1b[32m';
-const YELLOW_COLOR = '\x1b[33m';
-const RESET_COLOR = '\x1b[0m';
-
-const DASHBOARD_VERSION = 1;
-const DASHBOARD_FILENAME = 'dashboard.json';
-const DASHBOARD_MAX_BUILDS = 100;
-
-class FlakinessDashboard {
-  static async getCommitDetails(repoPath, ref = 'HEAD') {
-    const {stdout: timestamp} = await spawnAsyncOrDie('git', 'show', '-s', '--format=%ct', ref, {cwd: repoPath});
-    const {stdout: sha} = await spawnAsyncOrDie('git', 'rev-parse', ref, {cwd: repoPath});
-    return {timestamp: timestamp * 1000, sha: sha.trim()};
-  }
-
-  constructor({build, commit, dashboardRepo}) {
-    if (!commit)
-      throw new Error('"options.commit" must be specified!');
-    if (!commit.sha)
-      throw new Error('"options.commit.sha" must be specified!');
-    if (!commit.timestamp)
-      throw new Error('"options.commit.timestamp" must be specified!');
-    if (!build)
-      throw new Error('"options.build" must be specified!');
-    if (!build.url)
-      throw new Error('"options.build.url" must be specified!');
-    if (!dashboardRepo.branch)
-      throw new Error('"options.dashboardRepo.branch" must be specified!');
-    this._dashboardRepo = dashboardRepo;
-    this._build = new Build(Date.now(), build.url, commit, []);
-  }
-
-  reportTestResult(test) {
-    this._build._tests.push(test);
-  }
-
-  setBuildResult(result) {
-    this._build._result = result;
-  }
-
-  async uploadAndCleanup() {
-    console.log(`\n${YELLOW_COLOR}=== UPLOADING Flakiness Dashboard${RESET_COLOR}`);
-    const startTimestamp = Date.now();
-    const branch = this._dashboardRepo.branch.toLowerCase().replace(/\s/g, '-').replace(/[^-0-9a-zа-яё]/ig, '');
-    console.log(`  > Dashboard URL: ${this._dashboardRepo.url}`);
-    console.log(`  > Dashboard Branch: ${branch}`);
-    const git = await Git.initialize(this._dashboardRepo.url, branch, this._dashboardRepo.username, this._dashboardRepo.email, this._dashboardRepo.password);
-    console.log(`  > Dashboard Checkout: ${git.path()}`);
-
-    // Do at max 7 attempts to upload changes to github.
-    let success = false;
-    const MAX_ATTEMPTS = 7;
-    for (let i = 0; !success && i < MAX_ATTEMPTS; ++i) {
-      await saveBuildToDashboard(git.path(), this._build);
-      // if push went through - great! We're done!
-      if (await git.commitAllAndPush(`update dashboard\n\nbuild: ${this._build._url}`)) {
-        success = true;
-        console.log(`  > Push attempt ${YELLOW_COLOR}${i + 1}${RESET_COLOR} of ${YELLOW_COLOR}${MAX_ATTEMPTS}${RESET_COLOR}: ${GREEN_COLOR}SUCCESS${RESET_COLOR}`);
-      } else {
-        // Otherwise - wait random time between 3 and 11 seconds.
-        const cooldown = 3000 + Math.round(Math.random() * 1000) * 8;
-        console.log(`  > Push attempt ${YELLOW_COLOR}${i + 1}${RESET_COLOR} of ${YELLOW_COLOR}${MAX_ATTEMPTS}${RESET_COLOR}: ${RED_COLOR}FAILED${RESET_COLOR}, cooldown ${YELLOW_COLOR}${cooldown / 1000}${RESET_COLOR} seconds`);
-        await new Promise(x => setTimeout(x, cooldown));
-        // Reset our generated dashboard and pull from origin.
-        await git.hardResetToOriginMaster();
-        await git.pullFromOrigin();
-      }
-    }
-    await rmAsync(git.path());
-    console.log(`  > TOTAL TIME: ${YELLOW_COLOR}${(Date.now() - startTimestamp) / 1000}${RESET_COLOR} seconds`);
-    if (success)
-      console.log(`${YELLOW_COLOR}=== COMPLETE${RESET_COLOR}`);
-    else
-      console.log(`${RED_COLOR}=== FAILED${RESET_COLOR}`);
-    console.log('');
-  }
-}
-
-async function saveBuildToDashboard(dashboardPath, build) {
-  const filePath = path.join(dashboardPath, DASHBOARD_FILENAME);
-  let data = null;
-  try {
-    data = JSON.parse(await readFileAsync(filePath));
-  } catch (e) {
-    // Looks like there's no dashboard yet - create one.
-    data = {builds: []};
-  }
-  if (!data.builds)
-    throw new Error('Unrecognized dashboard format!');
-  data.builds.push({
-    version: DASHBOARD_VERSION,
-    result: build._result,
-    timestamp: build._timestamp,
-    url: build._url,
-    commit: build._commit,
-    tests: build._tests,
-  });
-  if (data.builds.length > DASHBOARD_MAX_BUILDS)
-    data.builds = data.builds.slice(data.builds.length - DASHBOARD_MAX_BUILDS);
-  await writeFileAsync(filePath, JSON.stringify(data));
-}
-
-class Build {
-  constructor(timestamp, url, commit, tests) {
-    this._timestamp = timestamp;
-    this._url = url;
-    this._commit = commit;
-    this._tests = tests;
-    this._result = undefined;
-  }
-}
-
-module.exports = {FlakinessDashboard};
-
-function promisify(nodeFunction) {
-  function promisified(...args) {
-    return new Promise((resolve, reject) => {
-      function callback(err, ...result) {
-        if (err)
-          return reject(err);
-        if (result.length === 1)
-          return resolve(result[0]);
-        return resolve(result);
-      }
-      nodeFunction.call(null, ...args, callback);
-    });
-  }
-  return promisified;
-}
-
-class Git {
-  static async initialize(url, branch, username, email, password) {
-    let schemeIndex = url.indexOf('://');
-    if (schemeIndex === -1)
-      throw new Error(`Malformed URL "${url}": expected to start with "https://"`);
-    schemeIndex += '://'.length;
-    url = url.substring(0, schemeIndex) + username + ':' + password + '@' + url.substring(schemeIndex);
-    const repoPath = await mkdtempAsync(TMP_FOLDER);
-    // Check existence of a remote branch for this bot.
-    const {stdout} = await spawnAsync('git', 'ls-remote', '--heads', url, branch);
-    // If there is no remote branch for this bot - create one.
-    if (!stdout.includes(branch)) {
-      await spawnAsyncOrDie('git', 'clone', '--no-checkout', '--depth=1', url, repoPath);
-
-      await spawnAsyncOrDie('git', 'checkout', '--orphan', branch, {cwd: repoPath});
-      await spawnAsyncOrDie('git', 'reset', '--hard', {cwd: repoPath});
-    } else {
-      await spawnAsyncOrDie('git', 'clone', '--single-branch', '--branch', `${branch}`, '--depth=1', url, repoPath);
-    }
-    await spawnAsyncOrDie('git', 'config', 'user.email', `"${email}"`, {cwd: repoPath});
-    await spawnAsyncOrDie('git', 'config', 'user.name', `"${username}"`, {cwd: repoPath});
-    return new Git(repoPath, url, branch, username);
-  }
-
-  async commitAllAndPush(message) {
-    await spawnAsyncOrDie('git', 'add', '.', {cwd: this._repoPath});
-    await spawnAsyncOrDie('git', 'commit', '-m', `${message}`, '--author', '"puppeteer-flakiness <aslushnikov+puppeteerflakiness@gmail.com>"', {cwd: this._repoPath});
-    const {code} = await spawnAsync('git', 'push', 'origin', this._branch, {cwd: this._repoPath});
-    return code === 0;
-  }
-
-  async hardResetToOriginMaster() {
-    await spawnAsyncOrDie('git', 'reset', '--hard', `origin/${this._branch}`, {cwd: this._repoPath});
-  }
-
-  async pullFromOrigin() {
-    await spawnAsyncOrDie('git', 'pull', 'origin', this._branch, {cwd: this._repoPath});
-  }
-
-  constructor(repoPath, url, branch, username) {
-    this._repoPath = repoPath;
-    this._url = url;
-    this._branch = branch;
-    this._username = username;
-  }
-
-  path() {
-    return this._repoPath;
-  }
-}
-
-async function spawnAsync(command, ...args) {
-  let options = {};
-  if (args.length && args[args.length - 1].constructor.name !== 'String')
-    options = args.pop();
-  const cmd = spawn(command, args, options);
-  let stdout = '';
-  let stderr = '';
-  cmd.stdout.on('data', data => stdout += data);
-  cmd.stderr.on('data', data => stderr += data);
-  const code = await new Promise(x => cmd.once('close', x));
-  if (stdout)
-    debug(stdout);
-  if (stderr)
-    debug(stderr);
-  return {code, stdout, stderr};
-}
-
-async function spawnAsyncOrDie(command, ...args) {
-  const {code, stdout, stderr} = await spawnAsync(command, ...args);
-  if (code !== 0)
-    throw new Error(`Failed to executed: "${command} ${args.join(' ')}".\n\n=== STDOUT ===\n${stdout}\n\n\n=== STDERR ===\n${stderr}`);
-  return {stdout, stderr};
-}

utils/flakiness-dashboard/index.js

@@ -1,3 +0,0 @@
-const {FlakinessDashboard} = require('./FlakinessDashboard');
-
-module.exports = {FlakinessDashboard};