From 890022e34779e303228515f465375f9dcbb13384 Mon Sep 17 00:00:00 2001 From: Claudio Wunder Date: Mon, 26 Oct 2026 03:01:17 +0100 Subject: [PATCH 1/9] perf: keep memory use down at low thread counts - Cap each worker's young generation at 32MB. V8 sizes it as if each worker had the whole machine, so it was over half of a worker's heap. - With one thread, process chunks one at a time, on a copy of their input as a worker gets. All chunks used to start at once, holding all their work together, and `jsx-ast` changed the metadata `legacy-json` reads, which only worked because `legacy-json` happened to finish first. - Let go of the pages' code once `html` compiled their programs: the generator and its caller both held it until every page was rendered. - Move `DEFAULT_MAX_THREADS` to the threading constants. Assisted-by: Claude Opus 5.5 --- packages/core/src/generators.mjs | 8 ++++--- packages/core/src/threading/constants.mjs | 14 +++++++++++ packages/core/src/threading/index.mjs | 10 ++++++-- packages/core/src/threading/parallel.mjs | 24 ++++++++++++++----- .../src/utils/configuration/constants.mjs | 7 ------ .../core/src/utils/configuration/index.mjs | 7 ++---- packages/react/src/html/generate.mjs | 14 +++++------ 7 files changed, 54 insertions(+), 30 deletions(-) diff --git a/packages/core/src/generators.mjs b/packages/core/src/generators.mjs index b73db096..b9959f4a 100644 --- a/packages/core/src/generators.mjs +++ b/packages/core/src/generators.mjs @@ -87,8 +87,10 @@ const createGenerator = () => { ? createParallelWorker(specifier, generator, pool, configuration) : Promise.resolve(null); - const result = await generate(dependencyInput, await worker); - + // Returned rather than awaited, so this function doesn't hold on to the + // input while the generator runs, and the generator can let it go + return generate(dependencyInput, await worker); + })().then(result => { // For streaming generators, "Completed" is logged when the cache // finishes collecting, not here when the generator returns if (!isAsyncIterable(result)) { @@ -96,7 +98,7 @@ const createGenerator = () => { } return result; - })() + }) ); }; diff --git a/packages/core/src/threading/constants.mjs b/packages/core/src/threading/constants.mjs index 82d46521..b307d49e 100644 --- a/packages/core/src/threading/constants.mjs +++ b/packages/core/src/threading/constants.mjs @@ -1,5 +1,12 @@ 'use strict'; +/** + * The default `threads` ceiling. Each worker holds a heap of its own, with the + * libraries and the pages it is working on, so past a few threads memory, not + * CPU, is what runs out. `--threads` raises it. + */ +export const DEFAULT_MAX_THREADS = 4; + /** * The heap size limit of each worker thread (V8's old space), in MB. V8 lets a * heap grow to several times its live data before collecting it, the more the @@ -8,3 +15,10 @@ * `--max-old-space-size` replaces it. */ export const WORKER_HEAP_SIZE = 512; + +/** + * The young generation size limit of each worker thread, in MB. V8 sizes it + * from the machine's memory, as if each worker were the only one, so it made up + * over half of a worker's heap. Node.js's `--max-semi-space-size` replaces it. + */ +export const WORKER_YOUNG_GENERATION_SIZE = 32; diff --git a/packages/core/src/threading/index.mjs b/packages/core/src/threading/index.mjs index a2db48c4..b70c94c8 100644 --- a/packages/core/src/threading/index.mjs +++ b/packages/core/src/threading/index.mjs @@ -2,7 +2,10 @@ import Piscina from 'piscina'; import logger from '#logger/index.mjs'; -import { WORKER_HEAP_SIZE } from './constants.mjs'; +import { + WORKER_HEAP_SIZE, + WORKER_YOUNG_GENERATION_SIZE, +} from './constants.mjs'; const poolLogger = logger.child('WorkerPool'); @@ -27,7 +30,10 @@ export default function createWorkerPool(threads) { // A worker idle for 250ms ends, so its heap is gone during a long stretch // of work on the main thread, such as bundling the site idleTimeout: 250, - resourceLimits: { maxOldGenerationSizeMb: WORKER_HEAP_SIZE }, + resourceLimits: { + maxOldGenerationSizeMb: WORKER_HEAP_SIZE, + maxYoungGenerationSizeMb: WORKER_YOUNG_GENERATION_SIZE, + }, recordTiming: false, workerData: { logLevel: logger.getLogLevel() }, }); diff --git a/packages/core/src/threading/parallel.mjs b/packages/core/src/threading/parallel.mjs index f8cfa312..caaa3821 100644 --- a/packages/core/src/threading/parallel.mjs +++ b/packages/core/src/threading/parallel.mjs @@ -96,17 +96,29 @@ export default function createParallelWorker( { generator: name, chunks: chunks.length, chunkSize, threads } ); - const runInOneGo = threads <= 1 || items.length <= 2; + // On this thread, one chunk at a time, so their work isn't all held at + // once, and on a copy, as a worker gets, which the chunk can change + if (threads <= 1 || items.length <= 2) { + for (const indices of chunks) { + const input = structuredClone(indices.map(i => items[i])); + + yield await generator.processChunk( + input, + input.keys().toArray(), + extra + ); + } + + return; + } // Start every chunk immediately, but emit results in submission order. // Handle errors as each task settles so a later failure cannot become an // unhandled rejection while earlier chunks are still running. const tasks = chunks.map(indices => { - const task = runInOneGo - ? generator.processChunk(items, indices, extra) - : pool.run( - createTask(items, indices, extra, configuration, specifier, name) - ); + const task = pool.run( + createTask(items, indices, extra, configuration, specifier, name) + ); return task.then( result => ({ result }), diff --git a/packages/core/src/utils/configuration/constants.mjs b/packages/core/src/utils/configuration/constants.mjs index b7ba3569..4c841e07 100644 --- a/packages/core/src/utils/configuration/constants.mjs +++ b/packages/core/src/utils/configuration/constants.mjs @@ -1,12 +1,5 @@ 'use strict'; -/** - * The default `threads` ceiling. Each worker holds a heap of its own, with the - * libraries and the pages it is working on, so past a few threads memory, not - * CPU, is what runs out. `--threads` raises it. - */ -export const DEFAULT_MAX_THREADS = 4; - /** * The default number of items each worker task processes. */ diff --git a/packages/core/src/utils/configuration/index.mjs b/packages/core/src/utils/configuration/index.mjs index 298b4877..9690e0be 100644 --- a/packages/core/src/utils/configuration/index.mjs +++ b/packages/core/src/utils/configuration/index.mjs @@ -22,11 +22,8 @@ import { import { resolveMarkdown } from '#utils/markdown/plugins.mjs'; import { deepMerge } from '#utils/misc.mjs'; -import { - DEFAULT_CHUNK_SIZE, - DEFAULT_MAX_THREADS, - DEFAULT_OUTPUT_DIRECTORY, -} from './constants.mjs'; +import { DEFAULT_MAX_THREADS } from '../../threading/constants.mjs'; +import { DEFAULT_CHUNK_SIZE, DEFAULT_OUTPUT_DIRECTORY } from './constants.mjs'; const configExplorer = cosmiconfig('doc-kit'); diff --git a/packages/react/src/html/generate.mjs b/packages/react/src/html/generate.mjs index 4f01f80e..83f7ba23 100644 --- a/packages/react/src/html/generate.mjs +++ b/packages/react/src/html/generate.mjs @@ -39,21 +39,20 @@ export async function generate(input, worker) { const template = await readFile(config.templatePath, 'utf-8'); - const pages = [...input]; - const all = config.generateAllPage ? buildAllPage(pages) : undefined; - // The built library and the compiled page programs live here until every // page is written; the directory is removed afterwards const outDir = await mkdtemp(join(tmpdir(), 'doc-kit-html-')); try { const { assets, tasks } = await buildPrograms({ - pages, - all, + pages: [...input], outDir, config, }); + // The pages' programs are compiled, so rendering doesn't need their code + input = undefined; + await createPageWriter(worker)(tasks, { template, assets }); } finally { await rm(outDir, { recursive: true, force: true }); @@ -71,12 +70,13 @@ export async function generate(input, worker) { * * @param {object} options * @param {Array} options.pages - Every page, in render order - * @param {import('./types').ComposedPage} [options.all] - `all.html`, when enabled * @param {string} options.outDir - Where the library and the programs are written * @param {import('./types').ResolvedWebConfiguration} options.config * @returns {Promise<{ assets: import('./types').ClientAssets, tasks: Array }>} */ -const buildPrograms = async ({ pages, all, outDir, config }) => { +const buildPrograms = async ({ pages, outDir, config }) => { + const all = config.generateAllPage ? buildAllPage(pages) : undefined; + // Every page's metadata, in render order — the sidebar, the index and the // cross links need the whole set. const datas = [...pages, ...(all ? [all] : [])].map(({ data }) => data); From fb523fa0c0bb32b68ddd3dd966e835056099df8b Mon Sep 17 00:00:00 2001 From: Claudio Wunder Date: Mon, 26 Oct 2026 03:57:59 +0100 Subject: [PATCH 2/9] perf(react): size the bundler's thread pools and allocator Vite's native bundler started a thread per CPU in rolldown's runtime and in the Rayon pool it shares with lightningcss, each holding memory of its own, and its allocator (mimalloc) kept what it freed: it only purges on its next call past a delay, which never comes once bundling ends, and with transparent huge pages it only returns whole free 2MB pages. On the Node.js docs that left ~800MB resident through rendering. The pools now get the threads doc-kit was given (`ROLLDOWN_WORKER_THREADS`, `RAYON_NUM_THREADS`; `TOKIO_WORKER_THREADS` and `ROLLDOWN_MAX_BLOCKING_THREADS` made no difference), and mimalloc returns memory as it's freed (`MIMALLOC_PURGE_DELAY=0`, `MIMALLOC_ALLOW_THP=0`). They're read when the bundler loads, so they're set just before it is, unless the environment already sets them. Assisted-by: Claude Opus 5.5 --- packages/react/src/html/bundlers/index.mjs | 18 ++++++++++++++++++ packages/react/src/html/constants.mjs | 11 +++++++++++ 2 files changed, 29 insertions(+) diff --git a/packages/react/src/html/bundlers/index.mjs b/packages/react/src/html/bundlers/index.mjs index 7b426ea1..198fc74c 100644 --- a/packages/react/src/html/bundlers/index.mjs +++ b/packages/react/src/html/bundlers/index.mjs @@ -1,3 +1,7 @@ +import getConfig from '@doc-kit/core/utils/configuration/index.mjs'; + +import { BUNDLER_THREAD_VARIABLES } from '../constants.mjs'; + /** * Returns the configured bundler or lazily creates the default Vite adapter. * @@ -9,6 +13,20 @@ export const resolveBundler = async bundler => { return bundler; } + // Vite's native thread pools (rolldown, lightningcss) otherwise start a + // thread per CPU, each holding memory of its own + const threads = String(getConfig().threads); + + for (const name of BUNDLER_THREAD_VARIABLES) { + process.env[name] ??= threads; + } + + // rolldown's allocator only returns freed memory on its next call past this + // delay, which never comes once bundling ends, so it returns it right away, + // and not in huge pages, which it only returns once they're entirely free + process.env.MIMALLOC_PURGE_DELAY ??= '0'; + process.env.MIMALLOC_ALLOW_THP ??= '0'; + const { createViteBundler } = await import('./vite.mjs'); return createViteBundler(); }; diff --git a/packages/react/src/html/constants.mjs b/packages/react/src/html/constants.mjs index 63cc661a..8919f1ee 100644 --- a/packages/react/src/html/constants.mjs +++ b/packages/react/src/html/constants.mjs @@ -92,3 +92,14 @@ export const FONTS = [ 'open-sans-latin-wght-italic.woff2', 'ibm-plex-mono-latin-400-normal.woff2', ]; + +/** + * The environment variables that size the thread pools of Vite's native + * bundler, read when they start: rolldown's runtime, and the Rayon pool + * rolldown and lightningcss share. + * @see https://rolldown.rs/guide/troubleshooting + */ +export const BUNDLER_THREAD_VARIABLES = [ + 'ROLLDOWN_WORKER_THREADS', + 'RAYON_NUM_THREADS', +]; From 1368640178b2098fed4fb75ff63eba9a7d0f9852 Mon Sep 17 00:00:00 2001 From: Claudio Wunder Date: Mon, 26 Oct 2026 04:10:54 +0100 Subject: [PATCH 3/9] meta: add changeset for the memory changes Assisted-by: Claude Opus 5.5 --- .changeset/keep-memory-down.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changeset/keep-memory-down.md diff --git a/.changeset/keep-memory-down.md b/.changeset/keep-memory-down.md new file mode 100644 index 00000000..6ecc8b02 --- /dev/null +++ b/.changeset/keep-memory-down.md @@ -0,0 +1,6 @@ +--- +'@doc-kit/core': patch +'@doc-kit/generator-react': patch +--- + +Builds use less memory, most of all with few threads: each worker's young generation is limited to 32MB, a single thread processes one chunk at a time and lets go of the pages' code before rendering them, and Vite's native bundler gets as many threads as doc-kit and returns the memory it frees. On the Node.js API docs, `--threads 1` now builds with Node.js's `--max-old-space-size=512`. From 95cef191ec0282d8211ca56abffcd8bfe7f52e76 Mon Sep 17 00:00:00 2001 From: Claudio Wunder Date: Mon, 26 Oct 2026 12:59:29 +0100 Subject: [PATCH 4/9] perf: only copy jsx-ast's input on the main thread Copying every chunk's input with one thread cost ~2.5s of a ~24s build, while only `jsx-ast` needs it: building a page changes its entries, and on the main thread those are the metadata other generators read. It now copies its own input there, and workers keep using theirs as is. Assisted-by: Claude Opus 5.5 --- packages/core/src/threading/parallel.mjs | 11 ++--------- packages/react/src/jsx-ast/generate.mjs | 8 +++++++- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/packages/core/src/threading/parallel.mjs b/packages/core/src/threading/parallel.mjs index caaa3821..136bac69 100644 --- a/packages/core/src/threading/parallel.mjs +++ b/packages/core/src/threading/parallel.mjs @@ -96,17 +96,10 @@ export default function createParallelWorker( { generator: name, chunks: chunks.length, chunkSize, threads } ); - // On this thread, one chunk at a time, so their work isn't all held at - // once, and on a copy, as a worker gets, which the chunk can change + // On this thread, one chunk at a time, so their work isn't all held at once if (threads <= 1 || items.length <= 2) { for (const indices of chunks) { - const input = structuredClone(indices.map(i => items[i])); - - yield await generator.processChunk( - input, - input.keys().toArray(), - extra - ); + yield await generator.processChunk(items, indices, extra); } return; diff --git a/packages/react/src/jsx-ast/generate.mjs b/packages/react/src/jsx-ast/generate.mjs index 05972ed1..62e51852 100644 --- a/packages/react/src/jsx-ast/generate.mjs +++ b/packages/react/src/jsx-ast/generate.mjs @@ -1,3 +1,5 @@ +import { isMainThread } from 'node:worker_threads'; + import getConfig from '@doc-kit/core/utils/configuration/index.mjs'; import { groupNodesByModule } from '@doc-kit/core/utils/generators.mjs'; import { jsx, toJs } from 'estree-util-to-js'; @@ -24,7 +26,11 @@ export async function processChunk(slicedInput, itemIndices) { const results = []; for (const idx of itemIndices) { - const { head, entries } = slicedInput[idx]; + // Building a page changes its entries. A worker has its own copy, but on + // the main thread they're the metadata other generators read too + const { head, entries } = isMainThread + ? structuredClone(slicedInput[idx]) + : slicedInput[idx]; const { content, ...page } = await buildContent(entries, head); From fe6db04312eaca2fb85dbb3e242d11fa04e431ce Mon Sep 17 00:00:00 2001 From: Claudio Wunder Date: Mon, 26 Oct 2026 12:59:38 +0100 Subject: [PATCH 5/9] perf: raise the worker young generation cap to 64MB At 32MB, workers collected their young generation often enough to cost 6-8% at 4 threads. 64MB keeps most of the memory saved (26-34% at 4 threads) within 1-2.5% of `main`'s build time. Assisted-by: Claude Opus 5.5 --- .changeset/keep-memory-down.md | 2 +- packages/core/src/threading/constants.mjs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.changeset/keep-memory-down.md b/.changeset/keep-memory-down.md index 6ecc8b02..a63c1fdd 100644 --- a/.changeset/keep-memory-down.md +++ b/.changeset/keep-memory-down.md @@ -3,4 +3,4 @@ '@doc-kit/generator-react': patch --- -Builds use less memory, most of all with few threads: each worker's young generation is limited to 32MB, a single thread processes one chunk at a time and lets go of the pages' code before rendering them, and Vite's native bundler gets as many threads as doc-kit and returns the memory it frees. On the Node.js API docs, `--threads 1` now builds with Node.js's `--max-old-space-size=512`. +Builds use less memory, most of all with few threads: each worker's young generation is limited to 64MB, a single thread processes one chunk at a time and lets go of the pages' code before rendering them, and Vite's native bundler gets as many threads as doc-kit and returns the memory it frees. On the Node.js API docs, `--threads 1` now builds with Node.js's `--max-old-space-size=512`. diff --git a/packages/core/src/threading/constants.mjs b/packages/core/src/threading/constants.mjs index b307d49e..b3d51e35 100644 --- a/packages/core/src/threading/constants.mjs +++ b/packages/core/src/threading/constants.mjs @@ -21,4 +21,4 @@ export const WORKER_HEAP_SIZE = 512; * from the machine's memory, as if each worker were the only one, so it made up * over half of a worker's heap. Node.js's `--max-semi-space-size` replaces it. */ -export const WORKER_YOUNG_GENERATION_SIZE = 32; +export const WORKER_YOUNG_GENERATION_SIZE = 64; From c6ce951e74315e4943c97c64a1406ac17a7d9726 Mon Sep 17 00:00:00 2001 From: Claudio Wunder Date: Mon, 26 Oct 2026 13:14:25 +0100 Subject: [PATCH 6/9] perf: raise the worker young generation cap to 96MB On 4 CPUs, like CI's runners, 64MB and 96MB keep the same memory (19% less across json, legacy-json, legacy-html, orama-db and web) and 96MB was marginally faster. It's also what V8 gives a worker with a 512MB old generation on Node.js 26, so the cap only changes Node.js 24. Assisted-by: Claude Opus 5.5 --- .changeset/keep-memory-down.md | 2 +- packages/core/src/threading/constants.mjs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.changeset/keep-memory-down.md b/.changeset/keep-memory-down.md index a63c1fdd..5d6a5127 100644 --- a/.changeset/keep-memory-down.md +++ b/.changeset/keep-memory-down.md @@ -3,4 +3,4 @@ '@doc-kit/generator-react': patch --- -Builds use less memory, most of all with few threads: each worker's young generation is limited to 64MB, a single thread processes one chunk at a time and lets go of the pages' code before rendering them, and Vite's native bundler gets as many threads as doc-kit and returns the memory it frees. On the Node.js API docs, `--threads 1` now builds with Node.js's `--max-old-space-size=512`. +Builds use less memory, most of all with few threads: each worker's young generation is limited to 96MB, a single thread processes one chunk at a time and lets go of the pages' code before rendering them, and Vite's native bundler gets as many threads as doc-kit and returns the memory it frees. On the Node.js API docs, `--threads 1` now builds with Node.js's `--max-old-space-size=512`. diff --git a/packages/core/src/threading/constants.mjs b/packages/core/src/threading/constants.mjs index b3d51e35..4a52d5f7 100644 --- a/packages/core/src/threading/constants.mjs +++ b/packages/core/src/threading/constants.mjs @@ -21,4 +21,4 @@ export const WORKER_HEAP_SIZE = 512; * from the machine's memory, as if each worker were the only one, so it made up * over half of a worker's heap. Node.js's `--max-semi-space-size` replaces it. */ -export const WORKER_YOUNG_GENERATION_SIZE = 64; +export const WORKER_YOUNG_GENERATION_SIZE = 96; From 30c2debfa23e9b6b3340300b42f1a95940e9b94d Mon Sep 17 00:00:00 2001 From: Claudio Wunder Date: Mon, 26 Oct 2026 13:20:43 +0100 Subject: [PATCH 7/9] refactor: rename WORKER_YOUNG_GENERATION_SIZE to WORKER_GEN_SIZE Assisted-by: Claude Opus 5.5 --- packages/core/src/threading/constants.mjs | 2 +- packages/core/src/threading/index.mjs | 7 ++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/packages/core/src/threading/constants.mjs b/packages/core/src/threading/constants.mjs index 4a52d5f7..9872c668 100644 --- a/packages/core/src/threading/constants.mjs +++ b/packages/core/src/threading/constants.mjs @@ -21,4 +21,4 @@ export const WORKER_HEAP_SIZE = 512; * from the machine's memory, as if each worker were the only one, so it made up * over half of a worker's heap. Node.js's `--max-semi-space-size` replaces it. */ -export const WORKER_YOUNG_GENERATION_SIZE = 96; +export const WORKER_GEN_SIZE = 96; diff --git a/packages/core/src/threading/index.mjs b/packages/core/src/threading/index.mjs index b70c94c8..0a2ed537 100644 --- a/packages/core/src/threading/index.mjs +++ b/packages/core/src/threading/index.mjs @@ -2,10 +2,7 @@ import Piscina from 'piscina'; import logger from '#logger/index.mjs'; -import { - WORKER_HEAP_SIZE, - WORKER_YOUNG_GENERATION_SIZE, -} from './constants.mjs'; +import { WORKER_HEAP_SIZE, WORKER_GEN_SIZE } from './constants.mjs'; const poolLogger = logger.child('WorkerPool'); @@ -32,7 +29,7 @@ export default function createWorkerPool(threads) { idleTimeout: 250, resourceLimits: { maxOldGenerationSizeMb: WORKER_HEAP_SIZE, - maxYoungGenerationSizeMb: WORKER_YOUNG_GENERATION_SIZE, + maxYoungGenerationSizeMb: WORKER_GEN_SIZE, }, recordTiming: false, workerData: { logLevel: logger.getLogLevel() }, From fbe874707d0bb75a80c4e9999714961f232ceb64 Mon Sep 17 00:00:00 2001 From: Claudio Wunder Date: Mon, 26 Oct 2026 13:26:06 +0100 Subject: [PATCH 8/9] fix(legacy-html): copy its input on the main thread Building a page splices the heading out of its entries' content and replaces the stability nodes, and on the main thread those entries are the metadata other generators read. With one thread, building `legacy-html` with other targets crashed (`Cannot read properties of undefined (reading 'tagName')`), on `main` too. It now works on a copy there, as it already does in a worker. Assisted-by: Claude Opus 5.5 --- .changeset/keep-memory-down.md | 3 ++- packages/node-legacy/src/legacy-html/generate.mjs | 7 ++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.changeset/keep-memory-down.md b/.changeset/keep-memory-down.md index 5d6a5127..e97ce569 100644 --- a/.changeset/keep-memory-down.md +++ b/.changeset/keep-memory-down.md @@ -1,6 +1,7 @@ --- '@doc-kit/core': patch '@doc-kit/generator-react': patch +'@node-core/doc-kit-legacy': patch --- -Builds use less memory, most of all with few threads: each worker's young generation is limited to 96MB, a single thread processes one chunk at a time and lets go of the pages' code before rendering them, and Vite's native bundler gets as many threads as doc-kit and returns the memory it frees. On the Node.js API docs, `--threads 1` now builds with Node.js's `--max-old-space-size=512`. +Builds use less memory, most of all with few threads: each worker's young generation is limited to 96MB, a single thread processes one chunk at a time and lets go of the pages' code before rendering them, and Vite's native bundler gets as many threads as doc-kit and returns the memory it frees. On the Node.js API docs, `--threads 1` now builds with Node.js's `--max-old-space-size=512`, and no longer crashes when `legacy-html` is built alongside other targets. diff --git a/packages/node-legacy/src/legacy-html/generate.mjs b/packages/node-legacy/src/legacy-html/generate.mjs index 5528b652..26c09fbb 100644 --- a/packages/node-legacy/src/legacy-html/generate.mjs +++ b/packages/node-legacy/src/legacy-html/generate.mjs @@ -2,6 +2,7 @@ import { readFile, cp } from 'node:fs/promises'; import { basename, join } from 'node:path'; +import { isMainThread } from 'node:worker_threads'; import getConfig from '@doc-kit/core/utils/configuration/index.mjs'; import { writeFile } from '@doc-kit/core/utils/file.mjs'; @@ -33,7 +34,11 @@ export async function processChunk(slicedInput, itemIndices, navigation) { const results = []; for (const idx of itemIndices) { - const { head, nodes, headNodes } = slicedInput[idx]; + // Building the content changes the entries. A worker has its own copy, but + // on the main thread they're the metadata other generators read too + const { head, nodes, headNodes } = isMainThread + ? structuredClone(slicedInput[idx]) + : slicedInput[idx]; const nav = navigation.replace( `class="nav-${head.api}"`, From aaaaa207f46cd8c35e32f1718ec44777704b36e0 Mon Sep 17 00:00:00 2001 From: Claudio Wunder Date: Mon, 26 Oct 2026 13:29:02 +0100 Subject: [PATCH 9/9] test: cover the memory changes - With one thread, chunks are processed one at a time, in order. - Workers get both the old and young generation limits. - jsx-ast and legacy-html leave their input unchanged on the main thread. - resolveBundler sets the bundler's thread and allocator variables from `threads`, keeps any already set, and leaves a custom bundler alone. Each fails against `main`. Assisted-by: Claude Opus 5.5 --- .../src/threading/__tests__/index.test.mjs | 14 ++++ .../src/threading/__tests__/parallel.test.mjs | 28 +++++++ .../legacy-html/__tests__/generate.test.mjs | 56 ++++++++++++++ .../html/bundlers/__tests__/index.test.mjs | 73 +++++++++++++++++++ .../src/jsx-ast/__tests__/generate.test.mjs | 11 +++ 5 files changed, 182 insertions(+) create mode 100644 packages/node-legacy/src/legacy-html/__tests__/generate.test.mjs create mode 100644 packages/react/src/html/bundlers/__tests__/index.test.mjs diff --git a/packages/core/src/threading/__tests__/index.test.mjs b/packages/core/src/threading/__tests__/index.test.mjs index dc6b63a3..20327884 100644 --- a/packages/core/src/threading/__tests__/index.test.mjs +++ b/packages/core/src/threading/__tests__/index.test.mjs @@ -4,6 +4,7 @@ import { fileURLToPath } from 'node:url'; import { LogLevel } from '../../logger/constants.mjs'; import logger from '../../logger/index.mjs'; +import { WORKER_GEN_SIZE, WORKER_HEAP_SIZE } from '../constants.mjs'; import createWorkerPool from '../index.mjs'; const reporterSpecifier = fileURLToPath( @@ -114,4 +115,17 @@ describe('createWorkerPool', () => { await pool.destroy(); } }); + + it("should limit each worker's old and young generation", async () => { + const pool = createWorkerPool(1); + + try { + deepStrictEqual(pool.options.resourceLimits, { + maxOldGenerationSizeMb: WORKER_HEAP_SIZE, + maxYoungGenerationSizeMb: WORKER_GEN_SIZE, + }); + } finally { + await pool.destroy(); + } + }); }); diff --git a/packages/core/src/threading/__tests__/parallel.test.mjs b/packages/core/src/threading/__tests__/parallel.test.mjs index 0667964b..4a2c3653 100644 --- a/packages/core/src/threading/__tests__/parallel.test.mjs +++ b/packages/core/src/threading/__tests__/parallel.test.mjs @@ -229,4 +229,32 @@ describe('createParallelWorker', () => { await rejects(result, /chunk failed/); }); + + it('should process one chunk at a time, in order, with a single thread', async () => { + let running = 0; + let mostRunning = 0; + + const generator = { + name: 'sequential', + async processChunk(items, indices) { + mostRunning = Math.max(mostRunning, ++running); + + await new Promise(resolve => setImmediate(resolve)); + + running--; + + return indices.map(index => items[index]); + }, + }; + + const worker = createParallelWorker('sequential', generator, null, { + threads: 1, + chunkSize: 2, + }); + + const chunks = await collectChunks(worker.stream([1, 2, 3, 4, 5], {})); + + strictEqual(mostRunning, 1); + deepStrictEqual(chunks, [[1, 2], [3, 4], [5]]); + }); }); diff --git a/packages/node-legacy/src/legacy-html/__tests__/generate.test.mjs b/packages/node-legacy/src/legacy-html/__tests__/generate.test.mjs new file mode 100644 index 00000000..39e9df70 --- /dev/null +++ b/packages/node-legacy/src/legacy-html/__tests__/generate.test.mjs @@ -0,0 +1,56 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; + +import { loadGenerator } from '@doc-kit/core/generators/loader.mjs'; +import { setConfig } from '@doc-kit/core/utils/configuration/index.mjs'; +import { loadMarkdownPlugins } from '@doc-kit/core/utils/markdown/plugins.mjs'; + +import { processChunk } from '../generate.mjs'; + +await loadMarkdownPlugins( + await loadGenerator(import.meta.resolve('../index.mjs')) +); + +const createEntry = (api, name) => { + const heading = { + type: 'heading', + depth: 1, + children: [{ type: 'text', value: name }], + data: { name, text: name, slug: api, type: 'module' }, + }; + + return { + api, + path: `/${api}`, + heading, + stability: null, + content: { + type: 'root', + children: [ + heading, + { + type: 'paragraph', + children: [{ type: 'text', value: `${name} body` }], + }, + ], + }, + }; +}; + +describe('legacy-html processChunk', () => { + it('does not change its input on the main thread', async () => { + await setConfig({ target: ['legacy-html'] }); + + const fs = createEntry('fs', 'File system'); + const before = structuredClone(fs); + + const [page] = await processChunk( + [{ head: fs, nodes: [fs], headNodes: [fs] }], + [0], + 'File system' + ); + + assert.match(page.content, /File system body/); + assert.deepEqual(fs, before); + }); +}); diff --git a/packages/react/src/html/bundlers/__tests__/index.test.mjs b/packages/react/src/html/bundlers/__tests__/index.test.mjs new file mode 100644 index 00000000..126a6080 --- /dev/null +++ b/packages/react/src/html/bundlers/__tests__/index.test.mjs @@ -0,0 +1,73 @@ +import assert from 'node:assert/strict'; +import { afterEach, describe, it, mock } from 'node:test'; + +import { setConfig } from '@doc-kit/core/utils/configuration/index.mjs'; + +mock.module('../vite.mjs', { + namedExports: { createViteBundler: () => 'vite' }, +}); + +const { resolveBundler } = await import('../index.mjs'); + +const VARIABLES = [ + 'ROLLDOWN_WORKER_THREADS', + 'RAYON_NUM_THREADS', + 'MIMALLOC_PURGE_DELAY', + 'MIMALLOC_ALLOW_THP', +]; + +const original = Object.fromEntries( + VARIABLES.map(name => [name, process.env[name]]) +); + +afterEach(() => { + for (const name of VARIABLES) { + if (original[name] === undefined) { + delete process.env[name]; + } else { + process.env[name] = original[name]; + } + } +}); + +describe('resolveBundler', () => { + it("sizes the bundler's thread pools and allocator before loading it", async () => { + await setConfig({ target: ['html'], threads: 3 }); + + for (const name of VARIABLES) { + delete process.env[name]; + } + + assert.equal(await resolveBundler(), 'vite'); + assert.equal(process.env.ROLLDOWN_WORKER_THREADS, '3'); + assert.equal(process.env.RAYON_NUM_THREADS, '3'); + assert.equal(process.env.MIMALLOC_PURGE_DELAY, '0'); + assert.equal(process.env.MIMALLOC_ALLOW_THP, '0'); + }); + + it('keeps what the environment already sets', async () => { + await setConfig({ target: ['html'], threads: 3 }); + + process.env.ROLLDOWN_WORKER_THREADS = '8'; + process.env.MIMALLOC_PURGE_DELAY = '250'; + + await resolveBundler(); + + assert.equal(process.env.ROLLDOWN_WORKER_THREADS, '8'); + assert.equal(process.env.MIMALLOC_PURGE_DELAY, '250'); + }); + + it('leaves the environment alone for a custom bundler', async () => { + for (const name of VARIABLES) { + delete process.env[name]; + } + + const bundler = {}; + + assert.equal(await resolveBundler(bundler), bundler); + + for (const name of VARIABLES) { + assert.equal(process.env[name], undefined); + } + }); +}); diff --git a/packages/react/src/jsx-ast/__tests__/generate.test.mjs b/packages/react/src/jsx-ast/__tests__/generate.test.mjs index c179fb96..d0324af4 100644 --- a/packages/react/src/jsx-ast/__tests__/generate.test.mjs +++ b/packages/react/src/jsx-ast/__tests__/generate.test.mjs @@ -108,4 +108,15 @@ describe('jsx-ast generate', () => { ['index', 'fs'] ); }); + + it('does not change its input on the main thread', async () => { + await setConfig({ target: ['jsx-ast'] }); + + const fs = createEntry('fs', 'File system'); + const before = structuredClone(fs); + + await processChunk([{ head: fs, entries: [fs] }], [0]); + + assert.deepEqual(fs, before); + }); });