From 251f2cc7b34fffde61563ea9ca94cbcc3ba2b891 Mon Sep 17 00:00:00 2001 From: Claudio Wunder Date: Sun, 15 Feb 2026 18:21:47 +0100 Subject: [PATCH 1/4] feat: introduce lazy generators --- src/generators.mjs | 13 +++-- src/generators/__tests__/index.test.mjs | 35 ++++++++---- .../api-links/__tests__/fixtures.test.mjs | 2 +- src/generators/index.mjs | 56 ++++++++----------- .../utils/__tests__/buildBarProps.test.mjs | 10 ++-- src/generators/types.d.ts | 11 ++++ src/threading/__tests__/parallel.test.mjs | 12 ++-- src/threading/chunk-worker.mjs | 2 +- src/threading/parallel.mjs | 4 +- .../configuration/__tests__/index.test.mjs | 6 +- src/utils/configuration/index.mjs | 47 ++++++++-------- src/utils/configuration/types.d.ts | 2 +- 12 files changed, 111 insertions(+), 89 deletions(-) diff --git a/src/generators.mjs b/src/generators.mjs index 69d29370..5245f534 100644 --- a/src/generators.mjs +++ b/src/generators.mjs @@ -48,16 +48,17 @@ const createGenerator = () => { * @param {string} generatorName - Generator to schedule * @param {import('./utils/configuration/types').Configuration} configuration - Runtime options */ - const scheduleGenerator = (generatorName, configuration) => { + const scheduleGenerator = async (generatorName, configuration) => { if (generatorName in cachedGenerators) { return; } - const { dependsOn, generate, processChunk } = allGenerators[generatorName]; + const { dependsOn, generate, processChunk } = + await allGenerators[generatorName](); // Schedule dependency first if (dependsOn && !(dependsOn in cachedGenerators)) { - scheduleGenerator(dependsOn, configuration); + await scheduleGenerator(dependsOn, configuration); } generatorsLogger.debug(`Scheduling "${generatorName}"`, { @@ -74,9 +75,9 @@ const createGenerator = () => { // Create parallel worker for streaming generators const worker = processChunk ? createParallelWorker(generatorName, pool, configuration) - : null; + : Promise.resolve(null); - const result = await generate(dependencyInput, worker); + const result = await generate(dependencyInput, await worker); // For streaming generators, "Completed" is logged when collection finishes // (in streamingCache.getOrCollect), not here when the generator returns @@ -107,7 +108,7 @@ const createGenerator = () => { // Schedule all generators for (const name of generators) { - scheduleGenerator(name, configuration); + await scheduleGenerator(name, configuration); } // Start all collections in parallel (don't await sequentially) diff --git a/src/generators/__tests__/index.test.mjs b/src/generators/__tests__/index.test.mjs index 1e456b0a..e95cf312 100644 --- a/src/generators/__tests__/index.test.mjs +++ b/src/generators/__tests__/index.test.mjs @@ -6,11 +6,23 @@ import semver from 'semver'; import { allGenerators } from '../index.mjs'; const validDependencies = Object.keys(allGenerators); -const generatorEntries = Object.entries(allGenerators); + +/** + * Resolves all lazy generator loaders into their actual metadata. + * @returns {Promise<[string, import('../types').GeneratorMetadata][]>} + */ +const resolveAllGenerators = async () => + Promise.all( + Object.entries(allGenerators).map(async ([key, loader]) => [ + key, + await loader(), + ]) + ); describe('All Generators', () => { - it('should have keys matching their name property', () => { - generatorEntries.forEach(([key, generator]) => { + it('should have keys matching their name property', async () => { + const entries = await resolveAllGenerators(); + entries.forEach(([key, generator]) => { assert.equal( key, generator.name, @@ -19,8 +31,9 @@ describe('All Generators', () => { }); }); - it('should have valid semver versions', () => { - generatorEntries.forEach(([key, generator]) => { + it('should have valid semver versions', async () => { + const entries = await resolveAllGenerators(); + entries.forEach(([key, generator]) => { const isValid = semver.valid(generator.version); assert.ok( isValid, @@ -29,8 +42,9 @@ describe('All Generators', () => { }); }); - it('should have valid dependsOn references', () => { - generatorEntries.forEach(([key, generator]) => { + it('should have valid dependsOn references', async () => { + const entries = await resolveAllGenerators(); + entries.forEach(([key, generator]) => { if (generator.dependsOn) { assert.ok( validDependencies.includes(generator.dependsOn), @@ -40,10 +54,11 @@ describe('All Generators', () => { }); }); - it('should have ast generator as a top-level generator with no dependencies', () => { - assert.ok(allGenerators.ast, 'ast generator should exist'); + it('should have ast generator as a top-level generator with no dependencies', async () => { + const ast = await allGenerators.ast(); + assert.ok(ast, 'ast generator should exist'); assert.equal( - allGenerators.ast.dependsOn, + ast.dependsOn, undefined, 'ast generator should have no dependencies' ); diff --git a/src/generators/api-links/__tests__/fixtures.test.mjs b/src/generators/api-links/__tests__/fixtures.test.mjs index 753b93ad..60e88208 100644 --- a/src/generators/api-links/__tests__/fixtures.test.mjs +++ b/src/generators/api-links/__tests__/fixtures.test.mjs @@ -35,7 +35,7 @@ describe('api links', () => { join(relativePath, 'fixtures', sourceFile).replaceAll(sep, '/'), ]; - const worker = createParallelWorker('ast-js', pool, config); + const worker = await createParallelWorker('ast-js', pool, config); // Collect results from the async generator const astJsResults = []; diff --git a/src/generators/index.mjs b/src/generators/index.mjs index 24294f92..7a67ba6c 100644 --- a/src/generators/index.mjs +++ b/src/generators/index.mjs @@ -1,44 +1,36 @@ 'use strict'; -import addonVerify from './addon-verify/index.mjs'; -import apiLinks from './api-links/index.mjs'; -import ast from './ast/index.mjs'; -import astJs from './ast-js/index.mjs'; -import jsonSimple from './json-simple/index.mjs'; -import jsxAst from './jsx-ast/index.mjs'; -import legacyHtml from './legacy-html/index.mjs'; -import legacyHtmlAll from './legacy-html-all/index.mjs'; -import legacyJson from './legacy-json/index.mjs'; -import legacyJsonAll from './legacy-json-all/index.mjs'; -import llmsTxt from './llms-txt/index.mjs'; -import manPage from './man-page/index.mjs'; -import metadata from './metadata/index.mjs'; -import oramaDb from './orama-db/index.mjs'; -import sitemap from './sitemap/index.mjs'; -import web from './web/index.mjs'; +/** + * Wraps a dynamic import into a lazy loader that resolves to the default export. + * + * @template T + * @param {() => Promise<{default: T}>} loader + * @returns {() => Promise} + */ +const lazyDefault = loader => () => loader().then(m => m.default); export const publicGenerators = { - 'json-simple': jsonSimple, - 'legacy-html': legacyHtml, - 'legacy-html-all': legacyHtmlAll, - 'man-page': manPage, - 'legacy-json': legacyJson, - 'legacy-json-all': legacyJsonAll, - 'addon-verify': addonVerify, - 'api-links': apiLinks, - 'orama-db': oramaDb, - 'llms-txt': llmsTxt, - sitemap, - web, + 'json-simple': lazyDefault(() => import('./json-simple/index.mjs')), + 'legacy-html': lazyDefault(() => import('./legacy-html/index.mjs')), + 'legacy-html-all': lazyDefault(() => import('./legacy-html-all/index.mjs')), + 'man-page': lazyDefault(() => import('./man-page/index.mjs')), + 'legacy-json': lazyDefault(() => import('./legacy-json/index.mjs')), + 'legacy-json-all': lazyDefault(() => import('./legacy-json-all/index.mjs')), + 'addon-verify': lazyDefault(() => import('./addon-verify/index.mjs')), + 'api-links': lazyDefault(() => import('./api-links/index.mjs')), + 'orama-db': lazyDefault(() => import('./orama-db/index.mjs')), + 'llms-txt': lazyDefault(() => import('./llms-txt/index.mjs')), + sitemap: lazyDefault(() => import('./sitemap/index.mjs')), + web: lazyDefault(() => import('./web/index.mjs')), }; // These ones are special since they don't produce standard output, // and hence, we don't expose them to the CLI. const internalGenerators = { - ast, - metadata, - 'jsx-ast': jsxAst, - 'ast-js': astJs, + ast: lazyDefault(() => import('./ast/index.mjs')), + metadata: lazyDefault(() => import('./metadata/index.mjs')), + 'jsx-ast': lazyDefault(() => import('./jsx-ast/index.mjs')), + 'ast-js': lazyDefault(() => import('./ast-js/index.mjs')), }; export const allGenerators = { diff --git a/src/generators/jsx-ast/utils/__tests__/buildBarProps.test.mjs b/src/generators/jsx-ast/utils/__tests__/buildBarProps.test.mjs index acef0965..8c296ea8 100644 --- a/src/generators/jsx-ast/utils/__tests__/buildBarProps.test.mjs +++ b/src/generators/jsx-ast/utils/__tests__/buildBarProps.test.mjs @@ -18,7 +18,7 @@ mock.module('../../../../utils/generators.mjs', { { version: '19.0.0', isLts: false, isCurrent: true }, ], leftHandAssign: Object.assign, - getVersionFromSemVer: version => version.split('.')[0], + getVersionFromSemVer: version => `${version.major}.x`, getVersionURL: (version, api) => `/api/${version}/${api}`, }, }); @@ -94,7 +94,7 @@ describe('buildMetaBarProps', () => { const result = buildMetaBarProps(head, entries); assert.equal(result.addedIn, 'v1.0.0'); - assert.equal(result.readingTime, '1 min read'); + assert.equal(result.readingTime, '5 min read'); assert.deepEqual(result.viewAs, [ ['JSON', 'fs.json'], ['MD', 'fs.md'], @@ -134,15 +134,15 @@ describe('formatVersionOptions', () => { assert.deepStrictEqual(result, [ { label: 'v16.x (LTS)', - value: 'https://nodejs.org/docs/latest-v16.x/api/http.html', + value: '/api/16.x/http', }, { label: 'v17.x (Current)', - value: 'https://nodejs.org/docs/latest-v17.x/api/http.html', + value: '/api/17.x/http', }, { label: 'v18.x', - value: 'https://nodejs.org/docs/latest-v18.x/api/http.html', + value: '/api/18.x/http', }, ]); }); diff --git a/src/generators/types.d.ts b/src/generators/types.d.ts index 43bc2cfd..bc7f24a8 100644 --- a/src/generators/types.d.ts +++ b/src/generators/types.d.ts @@ -1,12 +1,23 @@ import type { publicGenerators, allGenerators } from './index.mjs'; declare global { + /** + * A lazy generator loader that returns a promise resolving to the generator metadata. + */ + export type LazyGenerator> = + () => Promise; + // Public generators exposed to the CLI export type AvailableGenerators = typeof publicGenerators; // All generators including internal ones (metadata, jsx-ast, ast-js) export type AllGenerators = typeof allGenerators; + // The resolved type of a loaded generator + export type ResolvedGenerator = Awaited< + ReturnType + >; + /** * ParallelWorker interface for distributing work across Node.js worker threads. * Streams results as chunks complete, enabling pipeline parallelism. diff --git a/src/threading/__tests__/parallel.test.mjs b/src/threading/__tests__/parallel.test.mjs index 41b04c6f..758c3d8e 100644 --- a/src/threading/__tests__/parallel.test.mjs +++ b/src/threading/__tests__/parallel.test.mjs @@ -41,7 +41,7 @@ async function collectChunks(generator) { describe('createParallelWorker', () => { it('should create a ParallelWorker with stream method', async () => { const pool = createWorkerPool(2); - const worker = createParallelWorker('metadata', pool, { threads: 2 }); + const worker = await createParallelWorker('metadata', pool, { threads: 2 }); ok(worker); strictEqual(typeof worker.stream, 'function'); @@ -51,7 +51,7 @@ describe('createParallelWorker', () => { it('should handle empty items array', async () => { const pool = createWorkerPool(2); - const worker = createParallelWorker('ast-js', pool, { + const worker = await createParallelWorker('ast-js', pool, { threads: 2, chunkSize: 10, }); @@ -65,7 +65,7 @@ describe('createParallelWorker', () => { it('should distribute items to multiple worker threads', async () => { const pool = createWorkerPool(4); - const worker = createParallelWorker('metadata', pool, { + const worker = await createParallelWorker('metadata', pool, { threads: 4, chunkSize: 1, }); @@ -104,7 +104,7 @@ describe('createParallelWorker', () => { it('should yield results as chunks complete', async () => { const pool = createWorkerPool(2); - const worker = createParallelWorker('metadata', pool, { + const worker = await createParallelWorker('metadata', pool, { threads: 2, chunkSize: 1, }); @@ -131,7 +131,7 @@ describe('createParallelWorker', () => { it('should work with single thread and items', async () => { const pool = createWorkerPool(2); - const worker = createParallelWorker('metadata', pool, { + const worker = await createParallelWorker('metadata', pool, { threads: 2, chunkSize: 5, }); @@ -155,7 +155,7 @@ describe('createParallelWorker', () => { it('should use sliceInput for metadata generator', async () => { const pool = createWorkerPool(2); - const worker = createParallelWorker('metadata', pool, { + const worker = await createParallelWorker('metadata', pool, { threads: 2, chunkSize: 1, }); diff --git a/src/threading/chunk-worker.mjs b/src/threading/chunk-worker.mjs index 08e363ac..358ba8a1 100644 --- a/src/threading/chunk-worker.mjs +++ b/src/threading/chunk-worker.mjs @@ -17,7 +17,7 @@ export default async ({ }) => { await setConfig(configuration); - const generator = allGenerators[generatorName]; + const generator = await allGenerators[generatorName](); return generator.processChunk(input, itemIndices, extra); }; diff --git a/src/threading/parallel.mjs b/src/threading/parallel.mjs index 6627f0f0..a362c10c 100644 --- a/src/threading/parallel.mjs +++ b/src/threading/parallel.mjs @@ -63,14 +63,14 @@ const createTask = ( * @param {import('../utils/configuration/types').Configuration} configuration - Generator options * @returns {ParallelWorker} */ -export default function createParallelWorker( +export default async function createParallelWorker( generatorName, pool, configuration ) { const { threads, chunkSize } = configuration; - const generator = allGenerators[generatorName]; + const generator = await allGenerators[generatorName](); return { /** diff --git a/src/utils/configuration/__tests__/index.test.mjs b/src/utils/configuration/__tests__/index.test.mjs index 293a468f..229c5677 100644 --- a/src/utils/configuration/__tests__/index.test.mjs +++ b/src/utils/configuration/__tests__/index.test.mjs @@ -15,9 +15,9 @@ const createMockConfig = (overrides = {}) => ({ mock.module('../../../generators/index.mjs', { namedExports: { allGenerators: { - json: { defaultConfiguration: { format: 'json' } }, - html: { defaultConfiguration: { format: 'html' } }, - markdown: {}, + json: async () => ({ defaultConfiguration: { format: 'json' } }), + html: async () => ({ defaultConfiguration: { format: 'html' } }), + markdown: async () => ({}), }, }, }); diff --git a/src/utils/configuration/index.mjs b/src/utils/configuration/index.mjs index fd3d210d..7161b93a 100644 --- a/src/utils/configuration/index.mjs +++ b/src/utils/configuration/index.mjs @@ -14,30 +14,33 @@ import { importFromURL } from '../url.mjs'; /** * Get's the default configuration */ -export const getDefaultConfig = lazy(() => - Object.keys(allGenerators).reduce( - (acc, k) => { - acc[k] = allGenerators[k].defaultConfiguration ?? {}; - return acc; - }, - /** @type {import('./types').Configuration} */ ({ - global: { - version: process.version, - minify: true, +export const getDefaultConfig = lazy(async () => { + const defaults = /** @type {import('./types').Configuration} */ ({ + global: { + version: process.version, + minify: true, + repository: 'nodejs/node', + ref: 'HEAD', + baseURL: 'https://nodejs.org/docs', + changelog: populate(CHANGELOG_URL, { repository: 'nodejs/node', ref: 'HEAD', - baseURL: 'https://nodejs.org/docs', - changelog: populate(CHANGELOG_URL, { - repository: 'nodejs/node', - ref: 'HEAD', - }), - }, - - threads: cpus().length, - chunkSize: 10, + }), + }, + + threads: cpus().length, + chunkSize: 10, + }); + + await Promise.all( + Object.keys(allGenerators).map(async k => { + const generator = await allGenerators[k](); + defaults[k] = generator.defaultConfiguration ?? {}; }) - ) -); + ); + + return defaults; +}); /** * Loads a configuration file from a URL or file path. @@ -114,7 +117,7 @@ export const createRunConfiguration = async options => { const merged = deepMerge( config, createConfigFromCLIOptions(options), - getDefaultConfig() + await getDefaultConfig() ); // These need to be coerced diff --git a/src/utils/configuration/types.d.ts b/src/utils/configuration/types.d.ts index 853ad969..18deb134 100644 --- a/src/utils/configuration/types.d.ts +++ b/src/utils/configuration/types.d.ts @@ -16,7 +16,7 @@ export type Configuration = { chunkSize: number; } & { [K in keyof AllGenerators]: GlobalConfiguration & - AllGenerators[K]['defaultConfiguration']; + ResolvedGenerator['defaultConfiguration']; }; export type GlobalConfiguration = { From 54ffee14adaa4de716723ed5becc9b960b98be9c Mon Sep 17 00:00:00 2001 From: Claudio Wunder Date: Mon, 16 Feb 2026 00:08:20 +0100 Subject: [PATCH 2/4] chore: code review Co-authored-by: Aviv Keller --- src/generators/__tests__/index.test.mjs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/generators/__tests__/index.test.mjs b/src/generators/__tests__/index.test.mjs index e95cf312..4fffb321 100644 --- a/src/generators/__tests__/index.test.mjs +++ b/src/generators/__tests__/index.test.mjs @@ -7,12 +7,7 @@ import { allGenerators } from '../index.mjs'; const validDependencies = Object.keys(allGenerators); -/** - * Resolves all lazy generator loaders into their actual metadata. - * @returns {Promise<[string, import('../types').GeneratorMetadata][]>} - */ -const resolveAllGenerators = async () => - Promise.all( +const allGeneratorsReaolved = await Promise.all( Object.entries(allGenerators).map(async ([key, loader]) => [ key, await loader(), From 8815262a7518bb8fcc5171f0d51ae5cb33faad2f Mon Sep 17 00:00:00 2001 From: Claudio Wunder Date: Mon, 16 Feb 2026 01:08:14 +0100 Subject: [PATCH 3/4] fix: tests --- package.json | 1 + src/generators/__tests__/index.test.mjs | 19 ++++++++----------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/package.json b/package.json index 8d732d46..c65a665e 100644 --- a/package.json +++ b/package.json @@ -1,5 +1,6 @@ { "name": "@nodejs/doc-kit", + "type": "module", "repository": { "type": "git", "url": "git+https://github.com/nodejs/api-docs-tooling.git" diff --git a/src/generators/__tests__/index.test.mjs b/src/generators/__tests__/index.test.mjs index 4fffb321..bc9ef584 100644 --- a/src/generators/__tests__/index.test.mjs +++ b/src/generators/__tests__/index.test.mjs @@ -8,16 +8,15 @@ import { allGenerators } from '../index.mjs'; const validDependencies = Object.keys(allGenerators); const allGeneratorsReaolved = await Promise.all( - Object.entries(allGenerators).map(async ([key, loader]) => [ - key, - await loader(), - ]) - ); + Object.entries(allGenerators).map(async ([key, loader]) => [ + key, + await loader(), + ]) +); describe('All Generators', () => { it('should have keys matching their name property', async () => { - const entries = await resolveAllGenerators(); - entries.forEach(([key, generator]) => { + allGeneratorsReaolved.forEach(([key, generator]) => { assert.equal( key, generator.name, @@ -27,8 +26,7 @@ describe('All Generators', () => { }); it('should have valid semver versions', async () => { - const entries = await resolveAllGenerators(); - entries.forEach(([key, generator]) => { + allGeneratorsReaolved.forEach(([key, generator]) => { const isValid = semver.valid(generator.version); assert.ok( isValid, @@ -38,8 +36,7 @@ describe('All Generators', () => { }); it('should have valid dependsOn references', async () => { - const entries = await resolveAllGenerators(); - entries.forEach(([key, generator]) => { + allGeneratorsReaolved.forEach(([key, generator]) => { if (generator.dependsOn) { assert.ok( validDependencies.includes(generator.dependsOn), From b0c5469c7704cec6d1c6eef94efb3fc447c66c06 Mon Sep 17 00:00:00 2001 From: Claudio Wunder Date: Mon, 16 Feb 2026 13:39:25 +0100 Subject: [PATCH 4/4] chore: use lazy --- src/generators/index.mjs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/generators/index.mjs b/src/generators/index.mjs index 7a67ba6c..cb6e503b 100644 --- a/src/generators/index.mjs +++ b/src/generators/index.mjs @@ -1,5 +1,7 @@ 'use strict'; +import { lazy } from '../utils/misc.mjs'; + /** * Wraps a dynamic import into a lazy loader that resolves to the default export. * @@ -7,7 +9,7 @@ * @param {() => Promise<{default: T}>} loader * @returns {() => Promise} */ -const lazyDefault = loader => () => loader().then(m => m.default); +const lazyDefault = loader => lazy(() => loader().then(m => m.default)); export const publicGenerators = { 'json-simple': lazyDefault(() => import('./json-simple/index.mjs')),