diff --git a/.changeset/parallel-template-encode.md b/.changeset/parallel-template-encode.md new file mode 100644 index 0000000000..8902b445eb --- /dev/null +++ b/.changeset/parallel-template-encode.md @@ -0,0 +1,5 @@ +--- +"@lynx-js/template-webpack-plugin": patch +--- + +Run TASM template encoding in a shared `tinypool` worker pool so multi-entry builds encode in parallel and watch-mode rebuilds reuse warm workers. diff --git a/packages/webpack/template-webpack-plugin/etc/template-webpack-plugin.api.md b/packages/webpack/template-webpack-plugin/etc/template-webpack-plugin.api.md index 69b3445172..464a36dbfd 100644 --- a/packages/webpack/template-webpack-plugin/etc/template-webpack-plugin.api.md +++ b/packages/webpack/template-webpack-plugin/etc/template-webpack-plugin.api.md @@ -13,6 +13,7 @@ import * as CSS from '@lynx-js/css-serializer'; import { cssChunksToMap } from '@lynx-js/css-serializer'; import { Plugins } from '@lynx-js/css-serializer'; import { SyncWaterfallHook } from '@rspack/lite-tapable'; +import Tinypool from 'tinypool'; export { CSS } @@ -68,6 +69,7 @@ export class LynxEncodePlugin { static BEFORE_ENCODE_STAGE: number; static defaultOptions: Readonly>; static ENCODE_STAGE: number; + static encodePool: Tinypool; // (undocumented) protected options?: LynxEncodePluginOptions | undefined; } diff --git a/packages/webpack/template-webpack-plugin/package.json b/packages/webpack/template-webpack-plugin/package.json index 95744a1a7c..401739c46c 100644 --- a/packages/webpack/template-webpack-plugin/package.json +++ b/packages/webpack/template-webpack-plugin/package.json @@ -43,7 +43,8 @@ "@lynx-js/webpack-runtime-globals": "workspace:^", "@rspack/lite-tapable": "1.1.0", "css-tree": "^3.1.0", - "object.groupby": "^1.0.3" + "object.groupby": "^1.0.3", + "tinypool": "^2.1.0" }, "devDependencies": { "@lynx-js/test-tools": "workspace:*", @@ -56,6 +57,6 @@ "webpack": "^5.105.2" }, "engines": { - "node": ">=18" + "node": "^18.14 || >=19.4" } } diff --git a/packages/webpack/template-webpack-plugin/src/LynxEncodePlugin.ts b/packages/webpack/template-webpack-plugin/src/LynxEncodePlugin.ts index 3532ea12ad..2adcc9ded1 100644 --- a/packages/webpack/template-webpack-plugin/src/LynxEncodePlugin.ts +++ b/packages/webpack/template-webpack-plugin/src/LynxEncodePlugin.ts @@ -1,15 +1,26 @@ // Copyright 2024 The Lynx Authors. All rights reserved. // Licensed under the Apache License Version 2.0 that can be found in the // LICENSE file in the root directory of this source tree. +import { createRequire } from 'node:module'; +import { availableParallelism } from 'node:os'; +import { pathToFileURL } from 'node:url'; +import Tinypool from 'tinypool'; import type { Chunk, Compiler } from 'webpack'; +import type { EncodeResult } from '@lynx-js/tasm'; + import { collectCSSSourceMapContents, processTasmCSSDiagnostics, } from './cssDiagnostics.js'; import { LynxTemplatePlugin } from './LynxTemplatePlugin.js'; import { getRequireModuleAsyncCachePolyfill } from './polyfill/requireModuleAsync.js'; +import type { EncodeWorkerOptions } from './worker/encode.js'; + +const require = createRequire(import.meta.url); + +const ENCODE_WORKER_PATH = require.resolve('../lib/worker/encode.js'); // https://github.com/web-infra-dev/rsbuild/blob/main/packages/core/src/types/config.ts#L1029 type InlineChunkTestFunction = (params: { @@ -49,6 +60,17 @@ export class LynxEncodePlugin { * The stage of the beforeEmit hook. */ static BEFORE_EMIT_STAGE = 256; + /** + * Shared TASM encode worker pool: multiple entries (and multiple + * `LynxEncodePlugin` instances in the same process) share its worker + * slots for parallel encode; watch-mode rebuilds keep the same workers + * warm. `availableParallelism()` honors cgroup CPU limits (containers, + * CI runners), so we don't need to subtract a core ourselves. + */ + static encodePool: Tinypool = new Tinypool({ + filename: pathToFileURL(ENCODE_WORKER_PATH).href, + maxThreads: availableParallelism(), + }); constructor(protected options?: LynxEncodePluginOptions | undefined) {} /** @@ -228,17 +250,18 @@ export class LynxEncodePluginImpl { }, async (args) => { const { encodeOptions } = args; - const { getEncodeMode } = await import('@lynx-js/tasm'); - - const encode = getEncodeMode(); // TODO: lynx-js/tasm should add css_diagnostics type // @ts-expect-error ignore css_diagnostics type - const { buffer, lepus_debug, css_diagnostics } = await Promise.resolve( - encode(encodeOptions), + const { buffer, lepus_debug, css_diagnostics } = await ( + LynxEncodePlugin.encodePool.run( + { encodeOptions } as EncodeWorkerOptions, + ) as Promise ); return { - buffer, + // worker will serialize the buffer to a Uint8Array + // convert it back to a Buffer + buffer: Buffer.from(buffer), debugInfo: lepus_debug, cssDiagnostics: css_diagnostics as string, }; diff --git a/packages/webpack/template-webpack-plugin/src/worker/encode.ts b/packages/webpack/template-webpack-plugin/src/worker/encode.ts new file mode 100644 index 0000000000..9cc9d4b44e --- /dev/null +++ b/packages/webpack/template-webpack-plugin/src/worker/encode.ts @@ -0,0 +1,26 @@ +// Copyright 2026 The Lynx Authors. All rights reserved. +// Licensed under the Apache License Version 2.0 that can be found in the +// LICENSE file in the root directory of this source tree. +import type { EncodeResult } from '@lynx-js/tasm'; + +export interface EncodeWorkerOptions { + encodeBinary?: string | undefined; + encodeOptions: unknown; + tasmPkg?: string; +} + +export default async function encode( + { + encodeBinary = undefined, + encodeOptions, + tasmPkg = '@lynx-js/tasm', + }: EncodeWorkerOptions, +): Promise { + const { getEncodeMode } = + (await import(tasmPkg)) as typeof import('@lynx-js/tasm'); + // Napi will be used if supported + const encode = getEncodeMode(encodeBinary) as ( + options: unknown, + ) => Promise; + return encode(encodeOptions); +} diff --git a/packages/webpack/template-webpack-plugin/test/encode-worker-pool.test.ts b/packages/webpack/template-webpack-plugin/test/encode-worker-pool.test.ts new file mode 100644 index 0000000000..7a03833bb1 --- /dev/null +++ b/packages/webpack/template-webpack-plugin/test/encode-worker-pool.test.ts @@ -0,0 +1,96 @@ +// Copyright 2026 The Lynx Authors. All rights reserved. +// Licensed under the Apache License Version 2.0 that can be found in the +// LICENSE file in the root directory of this source tree. +import { dirname } from 'node:path'; + +import { describe, expect, test } from '@rstest/core'; +import webpack from 'webpack'; + +import { LynxEncodePlugin, LynxTemplatePlugin } from '../src/index.js'; + +const context = dirname(new URL(import.meta.url).pathname); + +function runWebpack(config: webpack.Configuration): Promise { + const compiler = webpack(config); + return new Promise((resolve, reject) => { + compiler.run((err, stats) => { + if (err) return reject(err); + if (!stats) return reject(new Error('webpack returned empty stats')); + resolve(stats); + compiler.close(() => void 0); + }); + }); +} + +describe('LynxEncodePlugin shared worker pool', () => { + // The static `LynxEncodePlugin.encodePool` is a process-wide singleton. + // These tests rely on rstest running each file in its own worker process, + // so the pool starts fresh for this file and the assertions below see + // monotonic state. + + test('runs multi-entry encodes in parallel via the shared pool', async () => { + const completedBefore = LynxEncodePlugin.encodePool.completed; + + const stats = await runWebpack({ + context, + mode: 'development', + devtool: false, + output: { iife: false, filename: '[name].js' }, + entry: { + a: './fixtures/basic.tsx', + b: './fixtures/basic.tsx', + }, + plugins: [ + new LynxTemplatePlugin(), + new LynxEncodePlugin(), + ], + }); + + expect(stats.compilation.errors).toEqual([]); + + // Two entries → two encode tasks went through the pool. + expect(LynxEncodePlugin.encodePool.completed - completedBefore).toBe(2); + + // Pool grew to (or already had) at least two threads to run them in + // parallel rather than serializing on a single worker. + expect(LynxEncodePlugin.encodePool.threads.length).toBeGreaterThanOrEqual( + 2, + ); + + const { assets } = stats.toJson({ all: false, assets: true }); + expect(assets?.find(i => i.name === 'a.js')).not.toBeUndefined(); + expect(assets?.find(i => i.name === 'b.js')).not.toBeUndefined(); + }); + + test('subsequent compile reuses warm workers (no respawn)', async () => { + const warmIds = new Set( + LynxEncodePlugin.encodePool.threads.map(t => t.threadId), + ); + expect(warmIds.size).toBeGreaterThan(0); + + const completedBefore = LynxEncodePlugin.encodePool.completed; + + // Simulate a watch-mode rebuild: a fresh compiler instance against the + // *same* process-wide pool. + await runWebpack({ + context, + mode: 'development', + devtool: false, + output: { iife: false, filename: '[name].js' }, + entry: { rebuild: './fixtures/basic.tsx' }, + plugins: [ + new LynxTemplatePlugin(), + new LynxEncodePlugin(), + ], + }); + + // The rebuild ran a task through the pool… + expect(LynxEncodePlugin.encodePool.completed - completedBefore).toBe(1); + + // …but every thread currently in the pool was already warm from the + // previous compile — none were newly spawned for the rebuild. + for (const { threadId } of LynxEncodePlugin.encodePool.threads) { + expect(warmIds.has(threadId)).toBe(true); + } + }); +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 911f4f26ac..811c3fefc4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2152,6 +2152,9 @@ importers: object.groupby: specifier: ^1.0.3 version: 1.0.3 + tinypool: + specifier: ^2.1.0 + version: 2.1.0 devDependencies: '@lynx-js/test-tools': specifier: workspace:* @@ -10691,6 +10694,10 @@ packages: resolution: {integrity: sha512-Zba82s87IFq9A9XmjiX5uZA/ARWDrB03OHlq+Vw1fSdt0I+4/Kutwy8BP4Y/y/aORMo61FQ0vIb5j44vSo5Pkg==} engines: {node: ^18.0.0 || >=20.0.0} + tinypool@2.1.0: + resolution: {integrity: sha512-Pugqs6M0m7Lv1I7FtxN4aoyToKg1C4tu+/381vH35y8oENM/Ai7f7C4StcoK4/+BSw9ebcS8jRiVrORFKCALLw==} + engines: {node: ^20.0.0 || >=22.0.0} + tinyrainbow@2.0.0: resolution: {integrity: sha512-op4nsTR47R6p0vMUUoYl/a+ljLFVtlfaXkLQmqfLR1qHma1h/ysYk4hEXZ880bf2CYgTskvTa/e196Vd5dDQXw==} engines: {node: '>=14.0.0'} @@ -20934,6 +20941,8 @@ snapshots: tinypool@1.1.1: {} + tinypool@2.1.0: {} + tinyrainbow@2.0.0: {} tinyspy@4.0.3: {}