[doclint] Prepare doclint for more checks

This patch refactors doclint so that more checks and more generators
could be added.

This patch:
- Introduces 'Source' class, which holds file content in-memory and
  allows it to be updated.
- Introduces 'Message' class - which is a pair of a text and a type.
  Messages could have either 'error' type or 'warning' type.

.eslintignore

@@ -1,3 +1,3 @@
 third_party/*
 examples/*
-utils/doclint/test/
+utils/doclint/check_public_api/test/

package.json

@@ -11,7 +11,7 @@
     "unit": "jasmine test/test.js",
     "debug-unit": "DEBUG_TEST=true node --inspect-brk ./node_modules/.bin/jasmine test/test.js",
     "test-phantom": "python third_party/phantomjs/test/run-tests.py",
-    "test-doclint": "jasmine utils/doclint/test/test.js",
+    "test-doclint": "jasmine utils/doclint/check_public_api/test/test.js",
     "test": "npm run lint --silent && npm run coverage && npm run test-phantom && npm run test-doclint",
     "install": "node install.js",
     "lint": "([ \"$CI\" = true ] && eslint --quiet -f codeframe . || eslint .) && npm run doc",

utils/doclint/Message.js

@@ -0,0 +1,44 @@
+/**
+ * Copyright 2017 Google Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+class Message {
+  /**
+   * @param {string} type
+   * @param {string} text
+   */
+  constructor(type, text) {
+    this.type = type;
+    this.text = text;
+  }
+
+  /**
+   * @param {string} text
+   * @return {!Message}
+   */
+  static error(text) {
+    return new Message('error', text);
+  }
+
+  /**
+   * @param {string} text
+   * @return {!Message}
+   */
+  static warning(text) {
+    return new Message('warning', text);
+  }
+}
+
+module.exports = Message;

utils/doclint/SourceFactory.js

@@ -0,0 +1,131 @@
+/**
+ * Copyright 2017 Google Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const path = require('path');
+const fs = require('fs');
+const {promisify} = require('util');
+const readFileAsync = promisify(fs.readFile);
+const readdirAsync = promisify(fs.readdir);
+const writeFileAsync = promisify(fs.writeFile);
+
+const PROJECT_DIR = path.join(__dirname, '..', '..');
+
+class Source {
+  /**
+   * @param {string} filePath
+   * @param {string} text
+   */
+  constructor(filePath, text) {
+    this._filePath = filePath;
+    this._projectPath = path.relative(PROJECT_DIR, filePath);
+    this._name = path.basename(filePath);
+    this._text = text;
+    this._hasUpdatedText = false;
+  }
+
+  /**
+   * @return {string}
+   */
+  filePath() {
+    return this._filePath;
+  }
+
+  /**
+   * @return {string}
+   */
+  projectPath() {
+    return this._projectPath;
+  }
+
+  /**
+   * @return {string}
+   */
+  name() {
+    return this._name;
+  }
+
+  /**
+   * @param {string} text
+   * @return {boolean}
+   */
+  setText(text) {
+    if (text === this._text)
+      return false;
+    this._hasUpdatedText = true;
+    this._text = text;
+    return true;
+  }
+
+  /**
+   * @return {string}
+   */
+  text() {
+    return this._text;
+  }
+
+  /**
+   * @return {boolean}
+   */
+  hasUpdatedText() {
+    return this._hasUpdatedText;
+  }
+}
+
+class SourceFactory {
+  constructor() {
+    this._sources = new Map();
+  }
+
+  /**
+   * @return {!Array<!Source>}
+   */
+  sources() {
+    return Array.from(this._sources.values());
+  }
+
+  async saveChangedSources() {
+    const changedSources = Array.from(this._sources.values()).filter(source => source.hasUpdatedText());
+    await Promise.all(changedSources.map(source => writeFileAsync(source.filePath(), source.text())));
+  }
+
+  /**
+   * @param {string} filePath
+   * @return {!Promise<Source>}
+   */
+  async readFile(filePath) {
+    filePath = path.resolve(filePath);
+    let source = this._sources.get(filePath);
+    if (!source) {
+      const text = await readFileAsync(filePath, {encoding: 'utf8'});
+      source = new Source(filePath, text);
+      this._sources.set(filePath, source);
+    }
+    return source;
+  }
+
+  /**
+   * @param {string} dirPath
+   * @param {string=} extension
+   * @return {!Promise<!Array<!Source>>}
+   */
+  async readdir(dirPath, extension = '') {
+    const fileNames = await readdirAsync(dirPath);
+    const filePaths = fileNames.filter(fileName => fileName.endsWith(extension)).map(fileName => path.join(dirPath, fileName));
+    return Promise.all(filePaths.map(filePath => this.readFile(filePath)));
+  }
+}
+
+module.exports = SourceFactory;

utils/doclint/check_public_api/JSBuilder.js

@@ -14,10 +14,8 @@
  * limitations under the License.
  */
 
-const fs = require('fs');
-const path = require('path');
 const esprima = require('esprima');
-const ESTreeWalker = require('../../third_party/chromium/ESTreeWalker');
+const ESTreeWalker = require('../../../third_party/chromium/ESTreeWalker');
 const Documentation = require('./Documentation');
 
 class JSOutline {
@@ -141,17 +139,14 @@ class JSOutline {
 }
 
 /**
- * @param {!Array<string>} dirPath
+ * @param {!Array<!Source>} sources
  * @return {!Promise<{documentation: !Documentation, errors: !Array<string>}>}
  */
-module.exports = async function(dirPath) {
-  let filePaths = fs.readdirSync(dirPath)
-      .filter(fileName => fileName.endsWith('.js'))
-      .map(fileName => path.join(dirPath, fileName));
+module.exports = async function(sources) {
   let classes = [];
   let errors = [];
-  for (let filePath of filePaths) {
-    let outline = new JSOutline(fs.readFileSync(filePath, 'utf8'));
+  for (let source of sources) {
+    let outline = new JSOutline(source.text());
     classes.push(...outline.classes);
     errors.push(...outline.errors);
   }

utils/doclint/check_public_api/MDBuilder.js

@@ -14,8 +14,6 @@
  * limitations under the License.
  */
 
-const fs = require('fs');
-const path = require('path');
 const Documentation = require('./Documentation');
 const commonmark = require('commonmark');
 
@@ -147,17 +145,14 @@ class MDOutline {
 
 /**
  * @param {!Page} page
- * @param {!Array<string>} dirPath
+ * @param {!Array<!Source>} sources
  * @return {!Promise<{documentation: !Documentation, errors: !Array<string>}>}
  */
-module.exports = async function(page, dirPath) {
-  let filePaths = fs.readdirSync(dirPath)
-      .filter(fileName => fileName.endsWith('.md'))
-      .map(fileName => path.join(dirPath, fileName));
+module.exports = async function(page, sources) {
   let classes = [];
   let errors = [];
-  for (let filePath of filePaths) {
-    let outline = await MDOutline.create(page, fs.readFileSync(filePath, 'utf8'));
+  for (let source of sources) {
+    let outline = await MDOutline.create(page, source.text());
     classes.push(...outline.classes);
     errors.push(...outline.errors);
   }

utils/doclint/check_public_api/index.js

@@ -17,6 +17,7 @@
 const jsBuilder = require('./JSBuilder');
 const mdBuilder = require('./MDBuilder');
 const Documentation = require('./Documentation');
+const Message = require('../Message');
 
 const EXCLUDE_CLASSES = new Set([
   'Connection',
@@ -47,13 +48,13 @@ const EXCLUDE_METHODS = new Set([
 
 /**
  * @param {!Page} page
- * @param {string} docsFolderPath
- * @param {string} jsFolderPath
- * @return {!Promise<!Array<string>>}
+ * @param {!Array<!Source>} mdSources
+ * @param {!Array<!Source>} jsSources
+ * @return {!Promise<!Array<!Message>>}
  */
-module.exports = async function lint(page, docsFolderPath, jsFolderPath) {
-  let mdResult = await mdBuilder(page, docsFolderPath);
-  let jsResult = await jsBuilder(jsFolderPath);
+module.exports = async function lint(page, mdSources, jsSources) {
+  let mdResult = await mdBuilder(page, mdSources);
+  let jsResult = await jsBuilder(jsSources);
   let jsDocumentation = filterJSDocumentation(jsResult.documentation);
   let mdDocumentation = mdResult.documentation;
 
@@ -68,7 +69,7 @@ module.exports = async function lint(page, docsFolderPath, jsFolderPath) {
   // Push all errors with proper prefixes
   let errors = jsErrors.map(error => '[JavaScript] ' + error);
   errors.push(...mdErrors.map(error => '[MarkDown] ' + error));
-  return errors;
+  return errors.map(error => Message.error(error));
 };
 
 /**

utils/doclint/check_public_api/test/test.js

@@ -17,9 +17,10 @@
 const fs = require('fs');
 const rm = require('rimraf').sync;
 const path = require('path');
-const Browser = require('../../../lib/Browser');
-const doclint = require('../lint');
-const GoldenUtils = require('../../../test/golden-utils');
+const Browser = require('../../../../lib/Browser');
+const checkPublicAPI = require('..');
+const SourceFactory = require('../../SourceFactory');
+const GoldenUtils = require('../../../../test/golden-utils');
 
 const OUTPUT_DIR = path.join(__dirname, 'output');
 const GOLDEN_DIR = path.join(__dirname, 'golden');
@@ -46,7 +47,7 @@ beforeEach(function() {
   GoldenUtils.addMatchers(jasmine, GOLDEN_DIR, OUTPUT_DIR);
 });
 
-describe('doclint', function() {
+describe('checkPublicAPI', function() {
   it('01-class-errors', SX(test));
   it('02-method-errors', SX(test));
   it('03-property-errors', SX(test));
@@ -58,8 +59,12 @@ describe('doclint', function() {
 });
 
 async function test() {
-  const filePath = path.join(__dirname, specName);
-  const errors = await doclint(page, filePath, filePath);
+  const dirPath = path.join(__dirname, specName);
+  const factory = new SourceFactory();
+  const mdSources = await factory.readdir(dirPath, '.md');
+  const jsSources = await factory.readdir(dirPath, '.js');
+  const messages = await checkPublicAPI(page, mdSources, jsSources);
+  const errors = messages.map(message => message.text);
   expect(errors.join('\n')).toBeGolden(specName + '.txt');
 }
 

utils/doclint/cli.js

@@ -17,62 +17,59 @@
 
 const Browser = require('../../lib/Browser');
 const path = require('path');
-const fs = require('fs');
-const lint = require('./lint');
-const markdownToc = require('markdown-toc');
+const SourceFactory = require('./SourceFactory');
 
 const PROJECT_DIR = path.join(__dirname, '..', '..');
-const DOCS_DIR = path.join(PROJECT_DIR, 'docs');
-const SOURCES_DIR = path.join(PROJECT_DIR, 'lib');
 
 const RED_COLOR = '\x1b[31m';
 const YELLOW_COLOR = '\x1b[33m';
 const RESET_COLOR = '\x1b[0m';
 
-const startTime = Date.now();
-const browser = new Browser({args: ['--no-sandbox']});
-browser.newPage().then(async page => {
-  const errors = await lint(page, DOCS_DIR, SOURCES_DIR);
-  await browser.close();
+run();
+
+async function run() {
+  const startTime = Date.now();
+
+  const sourceFactory = new SourceFactory();
+  /** @type {!Array<!Message>} */
+  const messages = [];
+
+  // Documentation checks.
+  {
+    const mdSources = await sourceFactory.readdir(path.join(PROJECT_DIR, 'docs'), '.md');
+    const jsSources = await sourceFactory.readdir(path.join(PROJECT_DIR, 'lib'), '.js');
+    const toc = require('./toc');
+    messages.push(...await toc(mdSources));
+
+    const browser = new Browser({args: ['--no-sandbox']});
+    const page = await browser.newPage();
+    const checkPublicAPI = require('./check_public_api');
+    messages.push(...await checkPublicAPI(page, mdSources, jsSources));
+    await browser.close();
+  }
+
+  // Report results.
+  const errors = messages.filter(message => message.type === 'error');
   if (errors.length) {
-    console.log('Documentation Failures:');
+    console.log('DocLint Failures:');
     for (let i = 0; i < errors.length; ++i) {
-      let error = errors[i];
+      let error = errors[i].text;
       error = error.split('\n').join('\n      ');
       console.log(`  ${i + 1}) ${RED_COLOR}${error}${RESET_COLOR}`);
     }
   }
-  const warnings = regenerateTOC(DOCS_DIR);
+  const warnings = messages.filter(message => message.type === 'warning');
   if (warnings.length) {
-    console.log('Documentation Warnings:');
+    console.log('DocLint Warnings:');
     for (let i = 0; i < warnings.length; ++i) {
-      let warning = warnings[i];
+      let warning = warnings[i].text;
       warning = warning.split('\n').join('\n      ');
       console.log(`  ${i + 1}) ${YELLOW_COLOR}${warning}${RESET_COLOR}`);
     }
   }
+  await sourceFactory.saveChangedSources();
   console.log(`${errors.length} failures, ${warnings.length} warnings.`);
   const runningTime = Date.now() - startTime;
-  console.log(`Finished in ${runningTime / 1000} seconds`);
+  console.log(`DocLint Finished in ${runningTime / 1000} seconds`);
   process.exit(errors.length + warnings.length > 0 ? 1 : 0);
-});
-
-/**
- * @param {string} dirPath
- * @return {!Array<string>}
- */
-function regenerateTOC(dirPath) {
-  let filePaths = fs.readdirSync(dirPath)
-      .filter(fileName => fileName.endsWith('.md'))
-      .map(fileName => path.join(dirPath, fileName));
-  let messages = [];
-  for (let filePath of filePaths) {
-    const markdownText = fs.readFileSync(filePath, 'utf8');
-    const newMarkdownText = markdownToc.insert(markdownText);
-    if (markdownText === newMarkdownText)
-      continue;
-    fs.writeFileSync(filePath, newMarkdownText, 'utf8');
-    messages.push('Regenerated table-of-contexts: ' + path.relative(PROJECT_DIR, filePath));
-  }
-  return messages;
 }

utils/doclint/toc.js

@@ -0,0 +1,32 @@
+/**
+ * Copyright 2017 Google Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const markdownToc = require('markdown-toc');
+const Message = require('./Message');
+
+/**
+ * @param {!Array<!Source>} sources
+ * @return {!Array<!Message>}
+ */
+module.exports = function(sources) {
+  const warnings = [];
+  for (let source of sources) {
+    const newText = markdownToc.insert(source.text());
+    if (source.setText(newText))
+      warnings.push('Regenerated table-of-contexts: ' + source.projectPath());
+  }
+  return warnings.map(warning => Message.warning(warning));
+};