[doclint] Implement simple markdown preprocessor

This patch implements simple markdown preprocessor. The goal
is to generate certain parts of markdown, such as:
- puppeteer version
- chromium revision
- table-of-contents
- copy/paste parts of documentation (for shortcut methods)

docs/api.md

@@ -1,4 +1,4 @@
-# Puppeteer API
+# Puppeteer API v<!-- GEN:version -->0.1.0<!-- GEN:stop-->
 
 ##### Table of Contents
 

package.json

@@ -11,7 +11,7 @@
     "unit": "jasmine test/test.js",
     "debug-unit": "DEBUG_TEST=true node --inspect-brk ./node_modules/.bin/jasmine test/test.js",
     "test-phantom": "python third_party/phantomjs/test/run-tests.py",
-    "test-doclint": "jasmine utils/doclint/check_public_api/test/test.js",
+    "test-doclint": "jasmine utils/doclint/check_public_api/test/test.js && jasmine utils/doclint/preprocessor/test.js",
     "test": "npm run lint --silent && npm run coverage && npm run test-phantom && npm run test-doclint",
     "install": "node install.js",
     "lint": "([ \"$CI\" = true ] && eslint --quiet -f codeframe . || eslint .) && npm run doc",

utils/doclint/SourceFactory.js

@@ -96,9 +96,15 @@ class SourceFactory {
     return Array.from(this._sources.values());
   }
 
+  /**
+   * @return {!Promise<boolean>}
+   */
   async saveChangedSources() {
     const changedSources = Array.from(this._sources.values()).filter(source => source.hasUpdatedText());
+    if (!changedSources.length)
+      return false;
     await Promise.all(changedSources.map(source => writeFileAsync(source.filePath(), source.text())));
+    return true;
   }
 
   /**
@@ -126,6 +132,15 @@ class SourceFactory {
     const filePaths = fileNames.filter(fileName => fileName.endsWith(extension)).map(fileName => path.join(dirPath, fileName));
     return Promise.all(filePaths.map(filePath => this.readFile(filePath)));
   }
+
+  /**
+   * @param {string} filePath
+   * @param {string} text
+   * @return {!Source}
+   */
+  createForTest(filePath, text) {
+    return new Source(filePath, text);
+  }
 }
 
 /**

utils/doclint/cli.js

@@ -37,13 +37,17 @@ async function run() {
   // Documentation checks.
   {
     const mdSources = await sourceFactory.readdir(path.join(PROJECT_DIR, 'docs'), '.md');
-    const jsSources = await sourceFactory.readdir(path.join(PROJECT_DIR, 'lib'), '.js');
+
     const toc = require('./toc');
     messages.push(...await toc(mdSources));
 
+    const preprocessor = require('./preprocessor');
+    messages.push(...await preprocessor(mdSources));
+
     const browser = new Browser({args: ['--no-sandbox']});
     const page = await browser.newPage();
     const checkPublicAPI = require('./check_public_api');
+    const jsSources = await sourceFactory.readdir(path.join(PROJECT_DIR, 'lib'), '.js');
     messages.push(...await checkPublicAPI(page, mdSources, jsSources));
     await browser.close();
   }
@@ -67,9 +71,14 @@ async function run() {
       console.log(`  ${i + 1}) ${YELLOW_COLOR}${warning}${RESET_COLOR}`);
     }
   }
-  await sourceFactory.saveChangedSources();
+  let clearExit = messages.length === 0;
+  if (await sourceFactory.saveChangedSources()) {
+    if (clearExit)
+      console.log(`${YELLOW_COLOR}Some files were updated.${RESET_COLOR}`);
+    clearExit = false;
+  }
   console.log(`${errors.length} failures, ${warnings.length} warnings.`);
   const runningTime = Date.now() - startTime;
   console.log(`DocLint Finished in ${runningTime / 1000} seconds`);
-  process.exit(errors.length + warnings.length > 0 ? 1 : 0);
+  process.exit(clearExit ? 0 : 1);
 }

utils/doclint/preprocessor/index.js

@@ -0,0 +1,65 @@
+/**
+ * Copyright 2017 Google Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const Message = require('../Message');
+
+const PUPPETEER_VERSION = require('../../../package.json').version;
+
+module.exports = function(sources) {
+  let messages = [];
+  let commands = [];
+  for (let source of sources) {
+    const text = source.text();
+    const commandStartRegex = /<!--\s*gen:([a-z]+)(?:\s*\(\s*([^)]*)\s*\))?\s*-->/ig;
+    const commandEndRegex = /<!--\s*gen:stop\s*-->/ig;
+    let start;
+
+    while (start = commandStartRegex.exec(text)) { // eslint-disable-line no-cond-assign
+      commandEndRegex.lastIndex = commandStartRegex.lastIndex;
+      const end = commandEndRegex.exec(text);
+      if (!end) {
+        messages.push(Message.error(`Failed to find 'gen:stop' for comamnd ${start[0]}`));
+        break;
+      }
+      commands.push({
+        name: start[1],
+        arg: start[2],
+        source: source,
+        from: commandStartRegex.lastIndex,
+        to: end.index,
+      });
+      commandStartRegex.lastIndex = commandEndRegex.lastIndex;
+    }
+  }
+  commands.sort((a, b) => b.from - a.from);
+  let changedSources = new Set();
+  for (let command of commands) {
+    let newText = command.source.text();
+    if (command.name === 'version')
+      newText = replaceInText(newText, command.from, command.to, PUPPETEER_VERSION);
+    else
+      errors.push(`Unknown GEN command in ${command.source.projectPath()}: ${command.name}`);
+    if (command.source.setText(newText))
+      changedSources.add(command.source);
+  }
+  for (source of changedSources)
+    messages.push(Message.warning(`GEN: updated ${source.projectPath()}`));
+  return messages;
+};
+
+function replaceInText(text, from, to, newText) {
+  return text.substring(0, from) + newText + text.substring(to);
+}

utils/doclint/preprocessor/test.js

@@ -0,0 +1,48 @@
+/**
+ * Copyright 2017 Google Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const preprocessor = require('.');
+const SourceFactory = require('../SourceFactory');
+const factory = new SourceFactory();
+const VERSION = require('../../../package.json').version;
+
+describe('preprocessor', function() {
+  describe('gen:version', function() {
+    it('should work', function() {
+      let source = factory.createForTest('doc.md', 'Puppeteer v<!--gen:version--><!--gen:stop-->');
+      let messages = preprocessor([source]);
+      expect(messages.length).toBe(1);
+      expect(messages[0].type).toBe('warning');
+      expect(messages[0].text).toContain('doc.md');
+      expect(source.text()).toBe(`Puppeteer v<!--gen:version-->${VERSION}<!--gen:stop-->`);
+    });
+    it('should tolerate different writing', function() {
+      let source = factory.createForTest('doc.md', `Puppeteer v<!--   gEn:version (  ) -->WHAT
+<!--     GEN:stop   -->`);
+      preprocessor([source]);
+      expect(source.text()).toBe(`Puppeteer v<!--   gEn:version (  ) -->${VERSION}<!--     GEN:stop   -->`);
+    });
+    it('should not tolerate missing gen:stop', function() {
+      let source = factory.createForTest('doc.md', `<!--GEN:version-->`);
+      let messages = preprocessor([source]);
+      expect(source.hasUpdatedText()).toBe(false);
+      expect(messages.length).toBe(1);
+      expect(messages[0].type).toBe('error');
+      expect(messages[0].text).toContain(`Failed to find 'gen:stop'`);
+    });
+  });
+});
+