diff --git a/.appveyor.yml b/.appveyor.yml index 5ca20300..0341893e 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -1,14 +1,10 @@ environment: matrix: - nodejs_version: "10.18.1" - FLAKINESS_DASHBOARD_NAME: Appveyor Chromium (Win + node10) - FLAKINESS_DASHBOARD_PASSWORD: - secure: g66jP+j6C+hkXLutBV9fdxB5fRJgcQQzy93SgQzXUmcCl/RjkJwnzyHvX0xfCVnv build: off install: - - ps: $env:FLAKINESS_DASHBOARD_BUILD_URL="https://ci.appveyor.com/project/aslushnikov/puppeteer/builds/$env:APPVEYOR_BUILD_ID/job/$env:APPVEYOR_JOB_ID" - ps: Install-Product node $env:nodejs_version - npm install - if "%nodejs_version%" == "10.18.1" ( diff --git a/.cirrus.yml b/.cirrus.yml index ba7737ff..8848a103 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -1,8 +1,5 @@ env: DISPLAY: :99.0 - FLAKINESS_DASHBOARD_PASSWORD: ENCRYPTED[b3e207db5d153b543f219d3c3b9123d8321834b783b9e45ac7d380e026ab3a56398bde51b521ac5859e7e45cb95d0992] - FLAKINESS_DASHBOARD_NAME: Cirrus ${CIRRUS_TASK_NAME} - FLAKINESS_DASHBOARD_BUILD_URL: https://cirrus-ci.com/task/${CIRRUS_TASK_ID} task: matrix: diff --git a/.travis.yml b/.travis.yml index b27051de..ae1b8cc1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,16 +31,10 @@ jobs: - node_js: "12.16.1" env: - CHROMIUM=true - - FLAKINESS_DASHBOARD_NAME="Travis Chromium (node12 + linux)" - - FLAKINESS_DASHBOARD_BUILD_URL="${TRAVIS_JOB_WEB_URL}" - node_js: "10.18.1" env: - CHROMIUM=true - - FLAKINESS_DASHBOARD_NAME="Travis Chromium (node10 + linux)" - - FLAKINESS_DASHBOARD_BUILD_URL="${TRAVIS_JOB_WEB_URL}" - node_js: "10.18.1" env: - FIREFOX=true - - FLAKINESS_DASHBOARD_NAME="Travis Firefox Nightly (node10 + linux)" - - FLAKINESS_DASHBOARD_BUILD_URL="${TRAVIS_JOB_WEB_URL}" - FIREFOX_HOME=$TRAVIS_HOME/firefox-latest diff --git a/test/test.js b/test/test.js index bcce2286..4f71c9b6 100644 --- a/test/test.js +++ b/test/test.js @@ -116,7 +116,6 @@ new Reporter(testRunner, { }); (async() => { - await utils.initializeFlakinessDashboardIfNeeded(testRunner); testRunner.run(); })(); diff --git a/test/utils.js b/test/utils.js index 952c61e0..052323b3 100644 --- a/test/utils.js +++ b/test/utils.js @@ -18,7 +18,6 @@ const fs = require('fs'); const path = require('path'); const expect = require('expect'); const GoldenUtils = require('./golden-utils'); -const {FlakinessDashboard} = require('../utils/flakiness-dashboard'); const PROJECT_ROOT = fs.existsSync(path.join(__dirname, '..', 'package.json')) ? path.join(__dirname, '..') : path.join(__dirname, '..', '..'); const COVERAGE_TESTSUITE_NAME = '**API COVERAGE**'; @@ -177,80 +176,4 @@ const utils = module.exports = { }); }); }, - - initializeFlakinessDashboardIfNeeded: async function(testRunner) { - // Generate testIDs for all tests and verify they don't clash. - // This will add |test.testId| for every test. - // - // NOTE: we do this on CI's so that problems arise on PR trybots. - if (process.env.CI) - generateTestIDs(testRunner); - // FLAKINESS_DASHBOARD_PASSWORD is an encrypted/secured variable. - // Encrypted variables get a special treatment in CI's when handling PRs so that - // secrets are not leaked to untrusted code. - // - AppVeyor DOES NOT decrypt secured variables for PRs - // - Travis DOES NOT decrypt encrypted variables for PRs - // - Cirrus CI DOES NOT decrypt encrypted variables for PRs *unless* PR is sent - // from someone who has WRITE ACCESS to the repo. - // - // Since we don't want to run flakiness dashboard for PRs on all CIs, we - // check existence of FLAKINESS_DASHBOARD_PASSWORD and absense of - // CIRRUS_BASE_SHA env variables. - if (!process.env.FLAKINESS_DASHBOARD_PASSWORD || process.env.CIRRUS_BASE_SHA) - return; - const {sha, timestamp} = await FlakinessDashboard.getCommitDetails(__dirname, 'HEAD'); - const dashboard = new FlakinessDashboard({ - commit: { - sha, - timestamp, - url: `https://github.com/puppeteer/puppeteer/commit/${sha}`, - }, - build: { - url: process.env.FLAKINESS_DASHBOARD_BUILD_URL, - }, - dashboardRepo: { - url: 'https://github.com/aslushnikov/puppeteer-flakiness-dashboard.git', - username: 'puppeteer-flakiness', - email: 'aslushnikov+puppeteerflakiness@gmail.com', - password: process.env.FLAKINESS_DASHBOARD_PASSWORD, - branch: process.env.FLAKINESS_DASHBOARD_NAME, - }, - }); - - testRunner.on('testfinished', test => { - // Do not report tests from COVERAGE testsuite. - // They don't bring much value to us. - if (test.fullName.includes(COVERAGE_TESTSUITE_NAME)) - return; - const testpath = test.location.filePath.substring(utils.projectRoot().length); - const url = `https://github.com/puppeteer/puppeteer/blob/${sha}/${testpath}#L${test.location.lineNumber}`; - dashboard.reportTestResult({ - testId: test.testId, - name: test.location.fileName + ':' + test.location.lineNumber, - description: test.fullName, - url, - result: test.result, - }); - }); - testRunner.on('finished', async({result}) => { - dashboard.setBuildResult(result); - await dashboard.uploadAndCleanup(); - }); - - function generateTestIDs(testRunner) { - const testIds = new Map(); - for (const test of testRunner.tests()) { - const testIdComponents = [test.name]; - for (let suite = test.suite; !!suite.parentSuite; suite = suite.parentSuite) - testIdComponents.push(suite.name); - testIdComponents.reverse(); - const testId = testIdComponents.join('>'); - const clashingTest = testIds.get(testId); - if (clashingTest) - throw new Error(`Two tests with clashing IDs: ${test.location.fileName}:${test.location.lineNumber} and ${clashingTest.location.fileName}:${clashingTest.location.lineNumber}`); - testIds.set(testId, test); - test.testId = testId; - } - } - }, }; diff --git a/utils/flakiness-dashboard/FlakinessDashboard.js b/utils/flakiness-dashboard/FlakinessDashboard.js deleted file mode 100644 index aa00d0e2..00000000 --- a/utils/flakiness-dashboard/FlakinessDashboard.js +++ /dev/null @@ -1,218 +0,0 @@ -const fs = require('fs'); -const os = require('os'); -const path = require('path'); -const spawn = require('child_process').spawn; -const debug = require('debug')('flakiness'); - -const rmAsync = promisify(require('rimraf')); -const mkdtempAsync = promisify(fs.mkdtemp); -const readFileAsync = promisify(fs.readFile); -const writeFileAsync = promisify(fs.writeFile); - -const TMP_FOLDER = path.join(os.tmpdir(), 'flakiness_tmp_folder-'); - -const RED_COLOR = '\x1b[31m'; -const GREEN_COLOR = '\x1b[32m'; -const YELLOW_COLOR = '\x1b[33m'; -const RESET_COLOR = '\x1b[0m'; - -const DASHBOARD_VERSION = 1; -const DASHBOARD_FILENAME = 'dashboard.json'; -const DASHBOARD_MAX_BUILDS = 100; - -class FlakinessDashboard { - static async getCommitDetails(repoPath, ref = 'HEAD') { - const {stdout: timestamp} = await spawnAsyncOrDie('git', 'show', '-s', '--format=%ct', ref, {cwd: repoPath}); - const {stdout: sha} = await spawnAsyncOrDie('git', 'rev-parse', ref, {cwd: repoPath}); - return {timestamp: timestamp * 1000, sha: sha.trim()}; - } - - constructor({build, commit, dashboardRepo}) { - if (!commit) - throw new Error('"options.commit" must be specified!'); - if (!commit.sha) - throw new Error('"options.commit.sha" must be specified!'); - if (!commit.timestamp) - throw new Error('"options.commit.timestamp" must be specified!'); - if (!build) - throw new Error('"options.build" must be specified!'); - if (!build.url) - throw new Error('"options.build.url" must be specified!'); - if (!dashboardRepo.branch) - throw new Error('"options.dashboardRepo.branch" must be specified!'); - this._dashboardRepo = dashboardRepo; - this._build = new Build(Date.now(), build.url, commit, []); - } - - reportTestResult(test) { - this._build._tests.push(test); - } - - setBuildResult(result) { - this._build._result = result; - } - - async uploadAndCleanup() { - console.log(`\n${YELLOW_COLOR}=== UPLOADING Flakiness Dashboard${RESET_COLOR}`); - const startTimestamp = Date.now(); - const branch = this._dashboardRepo.branch.toLowerCase().replace(/\s/g, '-').replace(/[^-0-9a-zа-яё]/ig, ''); - console.log(` > Dashboard URL: ${this._dashboardRepo.url}`); - console.log(` > Dashboard Branch: ${branch}`); - const git = await Git.initialize(this._dashboardRepo.url, branch, this._dashboardRepo.username, this._dashboardRepo.email, this._dashboardRepo.password); - console.log(` > Dashboard Checkout: ${git.path()}`); - - // Do at max 7 attempts to upload changes to github. - let success = false; - const MAX_ATTEMPTS = 7; - for (let i = 0; !success && i < MAX_ATTEMPTS; ++i) { - await saveBuildToDashboard(git.path(), this._build); - // if push went through - great! We're done! - if (await git.commitAllAndPush(`update dashboard\n\nbuild: ${this._build._url}`)) { - success = true; - console.log(` > Push attempt ${YELLOW_COLOR}${i + 1}${RESET_COLOR} of ${YELLOW_COLOR}${MAX_ATTEMPTS}${RESET_COLOR}: ${GREEN_COLOR}SUCCESS${RESET_COLOR}`); - } else { - // Otherwise - wait random time between 3 and 11 seconds. - const cooldown = 3000 + Math.round(Math.random() * 1000) * 8; - console.log(` > Push attempt ${YELLOW_COLOR}${i + 1}${RESET_COLOR} of ${YELLOW_COLOR}${MAX_ATTEMPTS}${RESET_COLOR}: ${RED_COLOR}FAILED${RESET_COLOR}, cooldown ${YELLOW_COLOR}${cooldown / 1000}${RESET_COLOR} seconds`); - await new Promise(x => setTimeout(x, cooldown)); - // Reset our generated dashboard and pull from origin. - await git.hardResetToOriginMaster(); - await git.pullFromOrigin(); - } - } - await rmAsync(git.path()); - console.log(` > TOTAL TIME: ${YELLOW_COLOR}${(Date.now() - startTimestamp) / 1000}${RESET_COLOR} seconds`); - if (success) - console.log(`${YELLOW_COLOR}=== COMPLETE${RESET_COLOR}`); - else - console.log(`${RED_COLOR}=== FAILED${RESET_COLOR}`); - console.log(''); - } -} - -async function saveBuildToDashboard(dashboardPath, build) { - const filePath = path.join(dashboardPath, DASHBOARD_FILENAME); - let data = null; - try { - data = JSON.parse(await readFileAsync(filePath)); - } catch (e) { - // Looks like there's no dashboard yet - create one. - data = {builds: []}; - } - if (!data.builds) - throw new Error('Unrecognized dashboard format!'); - data.builds.push({ - version: DASHBOARD_VERSION, - result: build._result, - timestamp: build._timestamp, - url: build._url, - commit: build._commit, - tests: build._tests, - }); - if (data.builds.length > DASHBOARD_MAX_BUILDS) - data.builds = data.builds.slice(data.builds.length - DASHBOARD_MAX_BUILDS); - await writeFileAsync(filePath, JSON.stringify(data)); -} - -class Build { - constructor(timestamp, url, commit, tests) { - this._timestamp = timestamp; - this._url = url; - this._commit = commit; - this._tests = tests; - this._result = undefined; - } -} - -module.exports = {FlakinessDashboard}; - -function promisify(nodeFunction) { - function promisified(...args) { - return new Promise((resolve, reject) => { - function callback(err, ...result) { - if (err) - return reject(err); - if (result.length === 1) - return resolve(result[0]); - return resolve(result); - } - nodeFunction.call(null, ...args, callback); - }); - } - return promisified; -} - -class Git { - static async initialize(url, branch, username, email, password) { - let schemeIndex = url.indexOf('://'); - if (schemeIndex === -1) - throw new Error(`Malformed URL "${url}": expected to start with "https://"`); - schemeIndex += '://'.length; - url = url.substring(0, schemeIndex) + username + ':' + password + '@' + url.substring(schemeIndex); - const repoPath = await mkdtempAsync(TMP_FOLDER); - // Check existence of a remote branch for this bot. - const {stdout} = await spawnAsync('git', 'ls-remote', '--heads', url, branch); - // If there is no remote branch for this bot - create one. - if (!stdout.includes(branch)) { - await spawnAsyncOrDie('git', 'clone', '--no-checkout', '--depth=1', url, repoPath); - - await spawnAsyncOrDie('git', 'checkout', '--orphan', branch, {cwd: repoPath}); - await spawnAsyncOrDie('git', 'reset', '--hard', {cwd: repoPath}); - } else { - await spawnAsyncOrDie('git', 'clone', '--single-branch', '--branch', `${branch}`, '--depth=1', url, repoPath); - } - await spawnAsyncOrDie('git', 'config', 'user.email', `"${email}"`, {cwd: repoPath}); - await spawnAsyncOrDie('git', 'config', 'user.name', `"${username}"`, {cwd: repoPath}); - return new Git(repoPath, url, branch, username); - } - - async commitAllAndPush(message) { - await spawnAsyncOrDie('git', 'add', '.', {cwd: this._repoPath}); - await spawnAsyncOrDie('git', 'commit', '-m', `${message}`, '--author', '"puppeteer-flakiness "', {cwd: this._repoPath}); - const {code} = await spawnAsync('git', 'push', 'origin', this._branch, {cwd: this._repoPath}); - return code === 0; - } - - async hardResetToOriginMaster() { - await spawnAsyncOrDie('git', 'reset', '--hard', `origin/${this._branch}`, {cwd: this._repoPath}); - } - - async pullFromOrigin() { - await spawnAsyncOrDie('git', 'pull', 'origin', this._branch, {cwd: this._repoPath}); - } - - constructor(repoPath, url, branch, username) { - this._repoPath = repoPath; - this._url = url; - this._branch = branch; - this._username = username; - } - - path() { - return this._repoPath; - } -} - -async function spawnAsync(command, ...args) { - let options = {}; - if (args.length && args[args.length - 1].constructor.name !== 'String') - options = args.pop(); - const cmd = spawn(command, args, options); - let stdout = ''; - let stderr = ''; - cmd.stdout.on('data', data => stdout += data); - cmd.stderr.on('data', data => stderr += data); - const code = await new Promise(x => cmd.once('close', x)); - if (stdout) - debug(stdout); - if (stderr) - debug(stderr); - return {code, stdout, stderr}; -} - -async function spawnAsyncOrDie(command, ...args) { - const {code, stdout, stderr} = await spawnAsync(command, ...args); - if (code !== 0) - throw new Error(`Failed to executed: "${command} ${args.join(' ')}".\n\n=== STDOUT ===\n${stdout}\n\n\n=== STDERR ===\n${stderr}`); - return {stdout, stderr}; -} diff --git a/utils/flakiness-dashboard/index.js b/utils/flakiness-dashboard/index.js deleted file mode 100644 index bb6ec923..00000000 --- a/utils/flakiness-dashboard/index.js +++ /dev/null @@ -1,3 +0,0 @@ -const {FlakinessDashboard} = require('./FlakinessDashboard'); - -module.exports = {FlakinessDashboard};