feat(json): add JSON generators

.changeset/json-generator.md

@@ -0,0 +1,7 @@
+---
+'@doc-kit/core': minor
+'@doc-kit/generator-react': patch
+'@node-core/doc-kit-legacy': patch
+---
+
+feat: the `json` and `json-all` generators

.github/workflows/generate.yml

@@ -94,6 +94,10 @@ jobs:
           - target: json-simple
             input: './node/doc/api/*.md'
 
+          - target: json
+            input: './node/doc/api/*.md'
+            compare: object-assertion
+
           - target: legacy-json
             input: './node/doc/api/*.md'
             compare: object-assertion

.prettierignore

@@ -16,6 +16,7 @@ www/out/
 
 # Generated Files
 packages/core/src/generators/metadata/maps/mdn.json
+packages/core/src/generators/json/generated/
 
 # The specification uses things that prettier would not
 # approve of, such as bullets with `*`

README.md

@@ -69,10 +69,11 @@ Options:
   --config-file <path>         Config file
   -i, --input <patterns...>    Input file patterns (glob)
   -t, --target <generator...>  Target generator(s): a built-in name
-                               (json-simple, legacy-html, legacy-html-all,
-                               man-page, legacy-json, legacy-json-all,
-                               addon-verify, api-links, orama-db, llms-txt,
-                               sitemap, html) or an import specifier for a
+                               (json, json-all, json-simple, legacy-html,
+                               legacy-html-all, man-page, legacy-json,
+                               legacy-json-all, addon-verify, api-links,
+                               orama-db, llms-txt, sitemap, html,
+                               section-pages) or an import specifier for a
                                custom generator
   --ignore <patterns...>       Ignore file patterns (glob)
   -o, --output <directory>     The output directory

docs/generators.md

@@ -20,9 +20,11 @@ npx @doc-kit/cli generate -t html -t orama-db -t sitemap -i "docs/**/*.md" -o ou
 
 ### JSON ([`@doc-kit/core`](./packages/core.md))
 
-| Target                                       | Output                                                   |
-| -------------------------------------------- | -------------------------------------------------------- |
-| [`json-simple`](./generators/json-simple.md) | A simplified JSON rendering of the parsed documentation. |
+| Target                                       | Output                                                             |
+| -------------------------------------------- | ------------------------------------------------------------------ |
+| [`json`](./generators/json.md)               | One schema-described JSON document per source file.                |
+| [`json-all`](./generators/json-all.md)       | Those documents bundled into a single `all.json`.                  |
+| [`json-simple`](./generators/json-simple.md) | A dump of the parsed metadata entries, for debugging the pipeline. |
 
 ### Legacy ([`@node-core/doc-kit-legacy`](./packages/node-legacy.md))
 

package.json

@@ -44,6 +44,7 @@
     "eslint-plugin-react-x": "5.18.1",
     "globals": "~17.7.0",
     "husky": "9.1.7",
+    "json-schema-to-typescript": "^16.0.0",
     "lint-staged": "17.3.0",
     "prettier": "3.9.6"
   }

packages/core/README.md

@@ -10,7 +10,8 @@ The command-line interface lives in the companion
 ## Generators
 
 Output formats are provided by generators. This package ships the shared
-pipeline stages and `json-simple`; the rest come from companion packages:
+pipeline stages and the JSON generators (`json`, `json-all`, and the
+debugging-only `json-simple`); the rest come from companion packages:
 
 | Package                                                                                | Generators                                                                            |
 | -------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------- |

packages/core/package.json

@@ -20,6 +20,8 @@
     ".": "./src/generators.mjs",
     "./ast": "./src/generators/ast/index.mjs",
     "./ast-js": "./src/generators/ast-js/index.mjs",
+    "./json": "./src/generators/json/index.mjs",
+    "./json-all": "./src/generators/json-all/index.mjs",
     "./json-simple": "./src/generators/json-simple/index.mjs",
     "./metadata": "./src/generators/metadata/index.mjs",
     "./package.json": "./package.json",
@@ -59,6 +61,8 @@
     "github-slugger": "^2.0.0",
     "glob-parent": "^6.0.2",
     "hastscript": "^9.0.1",
+    "mdast-util-slice-markdown": "^2.0.1",
+    "mdast-util-to-string": "^4.0.0",
     "piscina": "^5.3.0",
     "rehype-stringify": "^10.0.1",
     "remark-gfm": "^4.0.1",
@@ -79,13 +83,14 @@
     "yaml": "^2.9.0"
   },
   "devDependencies": {
+    "ajv": "^8.20.0",
     "hast-util-to-html": "^9.0.5",
     "hast-util-to-string": "^3.0.1"
   },
   "peerDependencies": {
+    "@doc-kit/generator-react": "workspace:>=0.1.0",
     "@node-core/doc-kit": "workspace:>=2.0.0",
-    "@node-core/doc-kit-legacy": "workspace:>=1.0.0",
-    "@doc-kit/generator-react": "workspace:>=0.1.0"
+    "@node-core/doc-kit-legacy": "workspace:>=1.0.0"
   },
   "peerDependenciesMeta": {
     "@node-core/doc-kit": {

packages/core/src/generators/index.mjs

@@ -10,6 +10,8 @@
  * which is how third-party generator packages are loaded.
  */
 export const publicGenerators = {
+  json: '@doc-kit/core/json',
+  'json-all': '@doc-kit/core/json-all',
   'json-simple': '@doc-kit/core/json-simple',
   'legacy-html': '@node-core/doc-kit-legacy/legacy-html',
   'legacy-html-all': '@node-core/doc-kit-legacy/legacy-html-all',

packages/core/src/generators/json-all/README.md

@@ -0,0 +1,31 @@
+# `json-all` Generator
+
+The `json-all` generator bundles the documents of the [`json`](./json.md)
+generator into a single `all.json` file.
+
+```sh
+npx @doc-kit/cli generate -t json-all -i "doc/api/*.md" -o out --index doc/api/index.md
+```
+
+```json
+{
+  "$schema": "https://doc-kit.nodejs.org/schemas/api-doc-all/1.0.0.json",
+  "documents": []
+}
+```
+
+`documents` holds every document in the order of the configured `index`,
+then the rest by `id`. The bundle's schema, shipped as
+`@doc-kit/core/generators/json-all/schema.json`, refers to the `json`
+generator's schema for the documents.
+
+## Configuring
+
+- `output` {string} The directory where `all.json` will be written.
+- `minify` {boolean} Whether to minify the output. Inherited from `global`.
+  **Default:** `true`.
+- `index` {Array} The `{ api }` objects defining the document order. Inherited
+  from `global`.
+- `schemaURL` {string} Where the bundle's schema is published.
+  `{schemaVersion}` is filled in. **Default:**
+  `'https://doc-kit.nodejs.org/schemas/api-doc-all/{schemaVersion}.json'`.

packages/core/src/generators/json-all/__tests__/generate.test.mjs

@@ -0,0 +1,34 @@
+import assert from 'node:assert/strict';
+import { describe, it } from 'node:test';
+
+import { setConfig } from '#utils/configuration/index.mjs';
+
+import { SCHEMA_VERSION } from '../../json/constants.mjs';
+import { generate } from '../generate.mjs';
+
+const config = await setConfig({ target: ['json-all'] });
+
+const document = id => ({ id, path: `/${id}`, children: [] });
+
+describe('json-all', () => {
+  it('bundles the documents in index order, then by id', async () => {
+    config['json-all'].index = [
+      { section: 'HTTP', api: 'http' },
+      { section: 'File system', api: 'fs' },
+    ];
+    config['json-all'].output = undefined;
+
+    const bundle = await generate(
+      ['zlib', 'fs', 'assert', 'http'].map(document)
+    );
+
+    assert.equal(
+      bundle.$schema,
+      `https://doc-kit.nodejs.org/schemas/api-doc-all/${SCHEMA_VERSION}.json`
+    );
+    assert.deepEqual(
+      bundle.documents.map(({ id }) => id),
+      ['http', 'fs', 'assert', 'zlib']
+    );
+  });
+});

packages/core/src/generators/json-all/constants.mjs

@@ -0,0 +1,5 @@
+'use strict';
+
+// Where a version of the bundle's schema is published.
+export const SCHEMA_URL =
+  'https://doc-kit.nodejs.org/schemas/api-doc-all/{schemaVersion}.json';

packages/core/src/generators/json-all/generate.mjs

@@ -0,0 +1,35 @@
+'use strict';
+
+import { join } from 'node:path';
+
+import getConfig from '#utils/configuration/index.mjs';
+import { writeJSON } from '#utils/file.mjs';
+
+import { resolveSchemaURL } from '../json/utils/schema.mjs';
+
+/**
+ * Bundles the `json` generator's documents into one `all.json` file.
+ *
+ * @type {import('./types').Generator['generate']}
+ */
+export async function generate(input) {
+  const config = getConfig('json-all');
+
+  // Documents follow the configured index; the rest go after it, by id
+  const order = new Map(config.index?.map(({ api }, i) => [api, i]));
+
+  const documents = input.toSorted(
+    (a, b) =>
+      (order.get(a.id) ?? Infinity) - (order.get(b.id) ?? Infinity) ||
+      a.id.localeCompare(b.id)
+  );
+
+  /** @type {import('./types').Bundle} */
+  const bundle = { $schema: resolveSchemaURL(config), documents };
+
+  if (config.output) {
+    await writeJSON(join(config.output, 'all.json'), bundle, config.minify);
+  }
+
+  return bundle;
+}

packages/core/src/generators/json-all/index.mjs

@@ -0,0 +1,25 @@
+'use strict';
+
+import { SCHEMA_URL } from './constants.mjs';
+import { generate } from './generate.mjs';
+
+/**
+ * This generator bundles the documents of the `json` generator into a single
+ * `all.json` file
+ *
+ * @type {import('./types').Generator}
+ */
+export default {
+  name: 'json-all',
+
+  description:
+    'Bundles the documents of the `json` generator into a single `all.json` file',
+
+  dependsOn: '@doc-kit/core/json',
+
+  defaultConfiguration: {
+    schemaURL: SCHEMA_URL,
+  },
+
+  generate,
+};

packages/core/src/generators/json-all/schema.json

@@ -0,0 +1,22 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://doc-kit.nodejs.org/schemas/api-doc-all/1.0.0.json",
+  "title": "Bundle",
+  "description": "Every document of a documentation set, as emitted by the doc-kit `json-all` generator, in index order.",
+  "type": "object",
+  "properties": {
+    "$schema": {
+      "type": "string",
+      "description": "The URL of the schema this bundle conforms to. Its last path segment is the schema version."
+    },
+    "documents": {
+      "type": "array",
+      "items": {
+        "$ref": "https://doc-kit.nodejs.org/schemas/api-doc/1.0.0.json"
+      },
+      "description": "The documents, in the order of the configured index, then by id."
+    }
+  },
+  "required": ["$schema", "documents"],
+  "additionalProperties": false
+}

packages/core/src/generators/json-all/types.d.ts

@@ -0,0 +1,20 @@
+import type { Document } from '../json/generated/schema';
+
+/**
+ * Every document of a documentation set, in index order.
+ */
+export interface Bundle {
+  /** The URL of the schema the bundle conforms to */
+  $schema: string;
+  documents: Array<Document>;
+}
+
+export interface Configuration {
+  /** Where the schema is published; `{schemaVersion}` is filled in */
+  schemaURL: string;
+}
+
+export type Generator = GeneratorMetadata<
+  Configuration,
+  Generate<Array<Document>, Promise<Bundle>>
+>;

packages/core/src/generators/json-simple/generate.mjs

@@ -5,7 +5,7 @@ import { join } from 'node:path';
 import { remove } from 'unist-util-remove';
 
 import getConfig from '#utils/configuration/index.mjs';
-import { writeFile } from '#utils/file.mjs';
+import { writeJSON } from '#utils/file.mjs';
 import { UNIST } from '#utils/queries/index.mjs';
 
 /**
@@ -23,10 +23,7 @@ export async function generate(input) {
   if (config.output) {
     // Writes all the API docs stringified content into one file
     // Note: The full JSON generator in the future will create one JSON file per top-level API doc file
-    await writeFile(
-      join(config.output, 'api-docs.json'),
-      config.minify ? JSON.stringify(input) : JSON.stringify(input, null, 2)
-    );
+    await writeJSON(join(config.output, 'api-docs.json'), input, config.minify);
   }
 
   return input;