Automatically generate table-of-contents for markdown

This patch teaches doclint to regenerate table of contents
automatically whenever it's needed.

This patch:
- splits lint.js into lint.js and cli.js
- teaches cli.js to generate table-of-contents
- removes the test for table-of-contents errors from doclint
- adds a test for doclint failing to parse object destructuring in
  method parameters.

package.json

@@ -14,8 +14,7 @@
     "test": "npm run lint --silent && npm run unit && npm run test-phantom && npm run test-doclint",
     "install": "node install.js",
     "lint": "([ \"$CI\" = true ] && eslint --quiet -f codeframe . || eslint .) && npm run doc",
-    "generate-toc": "markdown-toc -i docs/api.md",
+    "doc": "node utils/doclint/cli.js"
-    "doc": "node utils/doclint/lint.js"
   },
   "author": "The Chromium Authors",
   "license": "SEE LICENSE IN LICENSE",

utils/doclint/JSBuilder.js

@@ -74,7 +74,7 @@ class JSOutline {
       else if (param.type === 'Identifier')
         args.push(new Documentation.Argument(param.name));
       else
-        this.errors.push('JS Parsing issue: cannot support parameter of type ' + param.type + ' in method ' + methodName);
+        this.errors.push(`JS Parsing issue: unsupported syntax to define parameter in ${this._currentClassName}.${methodName}(): ${this._extractText(param)}`);
     }
     let method = Documentation.Member.createMethod(methodName, args, hasReturn, node.value.async);
     this._currentClassMembers.push(method);
@@ -126,17 +126,20 @@ class JSOutline {
 
 /**
  * @param {!Array<string>} dirPath
- * @return {!Promise<!Documentation>}
+ * @return {!Promise<{documentation: !Documentation, errors: !Array<string>}>}
  */
 module.exports = async function(dirPath) {
   let filePaths = fs.readdirSync(dirPath)
       .filter(fileName => fileName.endsWith('.js'))
       .map(fileName => path.join(dirPath, fileName));
   let classes = [];
+  let errors = [];
   for (let filePath of filePaths) {
     let outline = new JSOutline(fs.readFileSync(filePath, 'utf8'));
     classes.push(...outline.classes);
+    errors.push(...outline.errors);
   }
-  return new Documentation(classes);
+  const documentation = new Documentation(classes);
+  return { documentation, errors };
 };
 

utils/doclint/MDBuilder.js

@@ -1,5 +1,4 @@
 const fs = require('fs');
-const markdownToc = require('markdown-toc');
 const path = require('path');
 const Documentation = require('./Documentation');
 const commonmark = require('commonmark');
@@ -142,11 +141,7 @@ module.exports = async function(page, dirPath) {
   let classes = [];
   let errors = [];
   for (let filePath of filePaths) {
-    const markdownText = fs.readFileSync(filePath, 'utf8');
+    let outline = await MDOutline.create(page, fs.readFileSync(filePath, 'utf8'));
-    const newMarkdownText = markdownToc.insert(markdownText);
-    if (markdownText !== newMarkdownText)
-      errors.push('Markdown TOC is outdated, run `yarn generate-toc`');
-    let outline = await MDOutline.create(page, markdownText);
     classes.push(...outline.classes);
     errors.push(...outline.errors);
   }

utils/doclint/cli.js

@@ -0,0 +1,63 @@
+#!/usr/bin/env node
+
+const Browser = require('../../lib/Browser');
+const path = require('path');
+const fs = require('fs');
+const lint = require('./lint');
+const markdownToc = require('markdown-toc');
+
+const PROJECT_DIR = path.join(__dirname, '..', '..');
+const DOCS_DIR = path.join(PROJECT_DIR, 'docs');
+const SOURCES_DIR = path.join(PROJECT_DIR, 'lib');
+
+const RED_COLOR = '\x1b[31m';
+const YELLOW_COLOR = '\x1b[33m';
+const RESET_COLOR = '\x1b[0m';
+
+const startTime = Date.now();
+const browser = new Browser({args: ['--no-sandbox']});
+browser.newPage().then(async page => {
+  const errors = await lint(page, DOCS_DIR, SOURCES_DIR);
+  await browser.close();
+  if (errors.length) {
+    console.log('Documentation Failures:');
+    for (let i = 0; i < errors.length; ++i) {
+      let error = errors[i];
+      error = error.split('\n').join('\n      ');
+      console.log(`  ${i + 1}) ${RED_COLOR}${error}${RESET_COLOR}`);
+    }
+  }
+  const warnings = regenerateTOC(DOCS_DIR);
+  if (warnings.length) {
+    console.log('Documentation Warnings:');
+    for (let i = 0; i < warnings.length; ++i) {
+      let warning = warnings[i];
+      warning = warning.split('\n').join('\n      ');
+      console.log(`  ${i + 1}) ${YELLOW_COLOR}${warning}${RESET_COLOR}`);
+    }
+  }
+  console.log(`${errors.length} failures, ${warnings.length} warnings.`);
+  const runningTime = Date.now() - startTime;
+  console.log(`Finished in ${runningTime / 1000} seconds`);
+  process.exit(errors.length + warnings.length > 0 ? 1 : 0);
+});
+
+/**
+ * @param {string} dirPath
+ * @return {!Array<string>}
+ */
+function regenerateTOC(dirPath) {
+  let filePaths = fs.readdirSync(dirPath)
+      .filter(fileName => fileName.endsWith('.md'))
+      .map(fileName => path.join(dirPath, fileName));
+  let messages = [];
+  for (let filePath of filePaths) {
+    const markdownText = fs.readFileSync(filePath, 'utf8');
+    const newMarkdownText = markdownToc.insert(markdownText);
+    if (markdownText === newMarkdownText)
+      continue;
+    fs.writeFileSync(filePath, newMarkdownText, 'utf8');
+    messages.push('Regenerated table-of-contexts: ' + path.relative(PROJECT_DIR, filePath));
+  }
+  return messages;
+}

utils/doclint/lint.js

@@ -1,12 +1,8 @@
-const path = require('path');
 const jsBuilder = require('./JSBuilder');
 const mdBuilder = require('./MDBuilder');
 const Documentation = require('./Documentation');
-const Browser = require('../../lib/Browser');
 
-const PROJECT_DIR = path.join(__dirname, '..', '..');
+const EXCLUDE_CLASSES = new Set([
-
-let EXCLUDE_CLASSES = new Set([
   'AwaitedElement',
   'Connection',
   'EmulationManager',
@@ -18,7 +14,7 @@ let EXCLUDE_CLASSES = new Set([
   'TaskQueue',
 ]);
 
-let EXCLUDE_METHODS = new Set([
+const EXCLUDE_METHODS = new Set([
   'Body.constructor',
   'Dialog.constructor',
   'Frame.constructor',
@@ -41,15 +37,14 @@ let EXCLUDE_METHODS = new Set([
 async function lint(page, docsFolderPath, jsFolderPath) {
   let mdResult = await mdBuilder(page, docsFolderPath);
   let jsResult = await jsBuilder(jsFolderPath);
-  let jsDocumentation = filterJSDocumentation(jsResult);
+  let jsDocumentation = filterJSDocumentation(jsResult.documentation);
   let mdDocumentation = mdResult.documentation;
 
-  let jsErrors = [];
+  let jsErrors = jsResult.errors;
-  let mdErrors = Documentation.diff(mdDocumentation, jsDocumentation);
-  // Report all markdown parse errors.
-  mdErrors.push(...mdResult.errors);
-
   jsErrors.push(...Documentation.validate(jsDocumentation));
+
+  let mdErrors = mdResult.errors;
+  mdErrors.push(...Documentation.diff(mdDocumentation, jsDocumentation));
   mdErrors.push(...Documentation.validate(mdDocumentation));
   mdErrors.push(...lintMarkdown(mdDocumentation));
 
@@ -130,28 +125,3 @@ function filterJSDocumentation(jsDocumentation) {
 }
 
 module.exports = lint;
-
-const RED_COLOR = '\x1b[31m';
-const RESET_COLOR = '\x1b[0m';
-
-// Handle CLI invocation.
-if (!module.parent) {
-  const startTime = Date.now();
-  const browser = new Browser({args: ['--no-sandbox']});
-  browser.newPage().then(async page => {
-    const errors = await lint(page, path.join(PROJECT_DIR, 'docs'), path.join(PROJECT_DIR, 'lib'));
-    await browser.close();
-    if (errors.length) {
-      console.log('Documentation Failures:');
-      for (let i = 0; i < errors.length; ++i) {
-        let error = errors[i];
-        error = error.split('\n').join('\n    ');
-        console.log(`${i + 1}) ${RED_COLOR}${error}${RESET_COLOR}`);
-      }
-    }
-    console.log(`${errors.length} failures`);
-    const runningTime = Date.now() - startTime;
-    console.log(`Finished in ${runningTime / 1000} seconds`);
-    process.exit(errors.length > 0 ? 1 : 0);
-  });
-}

utils/doclint/test/04-bad-arguments/doc.md

@@ -3,5 +3,9 @@
 - `arg1` <[string]>
 - `arg2` <[string]>
 
+#### foo.bar(options)
+- `options` <[Object]>
+
 #### foo.test(...files)
 - `...filePaths` <[string]>
+

utils/doclint/test/04-bad-arguments/foo.js

@@ -4,4 +4,7 @@ class Foo {
 
   test(...filePaths) {
   }
+
+  bar({visibility}) {
+  }
 }

utils/doclint/test/05-outdated-toc/doc.md

@@ -1,8 +0,0 @@
-<!-- toc -->
-
-- [class: Dialog](#class-dialog)
-
-<!-- tocstop -->
-
-### class: Foo
-

utils/doclint/test/05-outdated-toc/foo.js

@@ -1,2 +0,0 @@
-class Foo {
-}

utils/doclint/test/golden/04-bad-arguments.txt

@@ -1,4 +1,7 @@
+[JavaScript] JS Parsing issue: unsupported syntax to define parameter in Foo.bar(): {visibility}
+[MarkDown] Heading arguments for "foo.test(...files)" do not match described ones, i.e. "...files" != "...filePaths"
+[MarkDown] Method Foo.bar() fails to describe its parameters:
+- Non-existing argument found: options
 [MarkDown] Method Foo.constructor() fails to describe its parameters:
 - Argument not found: arg3
 - Non-existing argument found: arg2
-[MarkDown] Heading arguments for "foo.test(...files)" do not match described ones, i.e. "...files" != "...filePaths"

utils/doclint/test/golden/05-outdated-toc.txt

@@ -1 +0,0 @@
-[MarkDown] Markdown TOC is outdated, run `yarn generate-toc`

utils/doclint/test/golden/08-return.txt

@@ -1,4 +1,4 @@
+[MarkDown] foo.www() has mistyped 'return' type declaration: expected exactly 'returns: ', found 'returns '.
 [MarkDown] Async method Foo.asyncFunction should describe return type Promise
 [MarkDown] Method Foo.return42 is missing return type description
 [MarkDown] Method Foo.returnNothing has unneeded description of return type
-[MarkDown] foo.www() has mistyped 'return' type declaration: expected exactly 'returns: ', found 'returns '.

utils/doclint/test/test.js

@@ -35,11 +35,10 @@ describe('doclint', function() {
   it('02-method-errors', SX(test));
   it('03-property-errors', SX(test));
   it('04-bad-arguments', SX(test));
-  it('05-outdated-toc', SX(test));
+  it('05-event-errors', SX(test));
   it('06-duplicates', SX(test));
   it('07-sorting', SX(test));
   it('08-return', SX(test));
-  it('09-event-errors', SX(test));
 });
 
 async function test() {