tools: build all.json by combining generated JSON

Notes:

1) Removed a number of root properties that did not seem relevant:
   source, desc, and introduced_in.  There no longer is a source, and
   the other two are from the first include and do not reflect the
   entire API.

2) As with https://github.com/nodejs/node/issues/20100, the current
   "desc" properties sometimes contained in-page links, other times
   referenced another page, and often did not match the links in the
   original HTML or JSON file. I chose to standardize on external links
   as "desc" values are isolated snippets as opposed to all.html which
   can be viewed as a standalone and self contained document.

3) Eliminated preprocessing for @include entirely, including the test
   case for this function.

4) _toc.md was renamed to index.md.

5) index comments no longer appear in embedded TOCs (left hand side
   column in the generated documentation.

PR-URL: https://github.com/nodejs/node/pull/21637
Reviewed-By: Vse Mozhet Byt <vsemozhetbyt@gmail.com>
Reviewed-By: Rich Trott <rtrott@gmail.com>

5 files changed, 66 insertions, 54 deletions

diff --git a/tools/doc/allhtml.js b/tools/doc/allhtml.js
index fad530f1c4..1c84e13d0a 100644
--- a/tools/doc/allhtml.js
+++ b/tools/doc/allhtml.js
@@ -12,7 +12,7 @@ const htmlFiles = fs.readdirSync(source, 'utf8')
   .filter((name) => name.includes('.html') && name !== 'all.html');
 
 // Read the table of contents.
-const toc = fs.readFileSync(source + '/_toc.html', 'utf8');
+const toc = fs.readFileSync(source + '/index.html', 'utf8');
 
 // Extract (and concatenate) the toc and apicontent from each document.
 let contents = '';
@@ -47,11 +47,12 @@ for (const link of toc.match(/<a.*?>/g)) {
   seen[href] = true;
 }
 
-// Replace various mentions of _toc with all.
-let all = toc.replace(/_toc\.html/g, 'all.html')
-  .replace('_toc.json', 'all.json')
-  .replace('api-section-_toc', 'api-section-all')
-  .replace('data-id="_toc"', 'data-id="all"');
+// Replace various mentions of index with all.
+let all = toc.replace(/index\.html/g, 'all.html')
+  .replace('<a href="all.html" name="toc">', '<a href="index.html" name="toc">')
+  .replace('index.json', 'all.json')
+  .replace('api-section-index', 'api-section-all')
+  .replace('data-id="index"', 'data-id="all"');
 
 // Clean up the title.
 all = all.replace(/<title>.*?\| /, '<title>');
diff --git a/tools/doc/alljson.js b/tools/doc/alljson.js
new file mode 100644
index 0000000000..7e027f764e
--- /dev/null
+++ b/tools/doc/alljson.js
@@ -0,0 +1,56 @@
+'use strict';
+
+// Build all.json by combining the miscs, modules, classes, globals, and methods
+// from the generated json files.
+
+const fs = require('fs');
+
+const source = `${__dirname}/../../out/doc/api`;
+
+// Get a list of generated API documents.
+const jsonFiles = fs.readdirSync(source, 'utf8')
+  .filter((name) => name.includes('.json') && name !== 'all.json');
+
+// Read the table of contents.
+const toc = fs.readFileSync(source + '/index.html', 'utf8');
+
+// Initialize results. Only these four data values will be collected.
+const results = {
+  miscs: [],
+  modules: [],
+  classes: [],
+  globals: [],
+  methods: []
+};
+
+// Identify files that should be skipped. As files are processed, they
+// are added to this list to prevent dupes.
+const seen = {
+  'all.json': true,
+  'index.json': true
+};
+
+// Extract (and concatenate) the selected data from each document.
+// Expand hrefs found in json to include source HTML file.
+for (const link of toc.match(/<a.*?>/g)) {
+  const href = /href="(.*?)"/.exec(link)[1];
+  const json = href.replace('.html', '.json');
+  if (!jsonFiles.includes(json) || seen[json]) continue;
+  const data = JSON.parse(
+    fs.readFileSync(source + '/' + json, 'utf8')
+      .replace(/<a href=\\"#/g, `<a href=\\"${href}#`)
+  );
+
+  for (const property in data) {
+    if (results.hasOwnProperty(property)) {
+      results[property].push(...data[property]);
+    }
+  }
+
+  // Mark source as seen.
+  seen[json] = true;
+}
+
+// Write results.
+fs.writeFileSync(source + '/all.json',
+                 `${JSON.stringify(results, null, 2)}\n`, 'utf8');
diff --git a/tools/doc/generate.js b/tools/doc/generate.js
index 9f217b19c7..8ed97610cf 100644
--- a/tools/doc/generate.js
+++ b/tools/doc/generate.js
@@ -21,7 +21,6 @@
 
 'use strict';
 
-const processIncludes = require('./preprocess.js');
 const fs = require('fs');
 
 // Parse the args.
@@ -53,12 +52,6 @@ if (!filename) {
 
 fs.readFile(filename, 'utf8', (er, input) => {
   if (er) throw er;
-  // Process the input for @include lines.
-  processIncludes(filename, input, next);
-});
-
-function next(er, input) {
-  if (er) throw er;
   switch (format) {
     case 'json':
       require('./json.js')(input, filename, (er, obj) => {
@@ -78,4 +71,4 @@ function next(er, input) {
     default:
       throw new Error(`Invalid format: ${format}`);
   }
-}
+});
diff --git a/tools/doc/html.js b/tools/doc/html.js
index 871a55baf4..0e254f1203 100644
--- a/tools/doc/html.js
+++ b/tools/doc/html.js
@@ -36,8 +36,8 @@ marked.setOptions({ renderer });
 
 const docPath = path.resolve(__dirname, '..', '..', 'doc');
 
-const gtocPath = path.join(docPath, 'api', '_toc.md');
-const gtocMD = fs.readFileSync(gtocPath, 'utf8').replace(/^@\/\/.*$/gm, '');
+const gtocPath = path.join(docPath, 'api', 'index.md');
+const gtocMD = fs.readFileSync(gtocPath, 'utf8').replace(/^<!--.*?-->/gms, '');
 const gtocHTML = marked(gtocMD).replace(
   /<a href="(.*?)"/g,
   (all, href) => `<a class="nav-${href.replace('.html', '')
diff --git a/tools/doc/preprocess.js b/tools/doc/preprocess.js
deleted file mode 100644
index 554af2ccb7..0000000000
--- a/tools/doc/preprocess.js
+++ /dev/null
@@ -1,38 +0,0 @@
-'use strict';
-
-module.exports = processIncludes;
-
-const path = require('path');
-const fs = require('fs');
-
-const includeExpr = /^@include\s+([\w-]+)(?:\.md)?$/gmi;
-const commentExpr = /^@\/\/.*$/gm;
-
-function processIncludes(inputFile, input, cb) {
-  const includes = input.match(includeExpr);
-  if (includes === null)
-    return cb(null, input.replace(commentExpr, ''));
-
-  let errState = null;
-  let incCount = includes.length;
-
-  includes.forEach((include) => {
-    const fname = include.replace(includeExpr, '$1.md');
-    const fullFname = path.resolve(path.dirname(inputFile), fname);
-
-    fs.readFile(fullFname, 'utf8', function(er, inc) {
-      if (errState) return;
-      if (er) return cb(errState = er);
-      incCount--;
-
-      // Add comments to let the HTML generator know
-      // how the anchors for headings should look like.
-      inc = `<!-- [start-include:${fname}] -->\n` +
-            `${inc}\n<!-- [end-include:${fname}] -->\n`;
-      input = input.split(`${include}\n`).join(`${inc}\n`);
-
-      if (incCount === 0)
-        return cb(null, input.replace(commentExpr, ''));
-    });
-  });
-}