Skip to content

Commit

Permalink
refactor: only run webpack once for multi-arch packages (#3437)
Browse files Browse the repository at this point in the history 
* refactor: only run webpack once for multi-arch packages

* fix: handle comma seperated arch string
  • Loading branch information
@MarshallOfSound
MarshallOfSound authored Dec 6, 2023
1 parent c56f406 commit 6e8e1ed
Show file tree
Hide file tree
Showing 6 changed files with 192 additions and 59 deletions.
2 changes: 1 addition & 1 deletion packages/plugin/base/src/Plugin.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ export default abstract class Plugin<C> implements IForgePlugin {
// This method is not type safe internally, but is type safe for consumers
// @internal
export const namedHookWithTaskFn = <Hook extends ForgeHookName>(
hookFn: (task: ForgeListrTask<never> | null, ...args: Parameters<ForgeHookFn<Hook>>) => ReturnType<ForgeHookFn<Hook>>,
hookFn: <Ctx = never>(task: ForgeListrTask<Ctx> | null, ...args: Parameters<ForgeHookFn<Hook>>) => ReturnType<ForgeHookFn<Hook>>,
name: string
): ForgeHookFn<Hook> => {
function namedHookWithTaskInner(this: ForgeListrTask<any> | null, ...args: any[]) {
Expand Down
1 change: 1 addition & 0 deletions packages/plugin/webpack/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
"@electron-forge/web-multi-logger": "7.2.0",
"chalk": "^4.0.0",
"debug": "^4.3.1",
"fast-glob": "^3.2.7",
"fs-extra": "^10.0.0",
"html-webpack-plugin": "^5.5.3",
"webpack": "^5.69.1",
Expand Down
237 changes: 184 additions & 53 deletions packages/plugin/webpack/src/WebpackPlugin.ts
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,15 @@
import crypto from 'crypto';
import http from 'http';
import path from 'path';
import { pipeline } from 'stream/promises';

import { getElectronVersion, listrCompatibleRebuildHook } from '@electron-forge/core-utils';
import { namedHookWithTaskFn, PluginBase } from '@electron-forge/plugin-base';
import { ForgeListrTaskDefinition, ForgeMultiHookMap, ResolvedForgeConfig, StartResult } from '@electron-forge/shared-types';
import { ForgeMultiHookMap, ListrTask, ResolvedForgeConfig, StartResult } from '@electron-forge/shared-types';
import Logger, { Tab } from '@electron-forge/web-multi-logger';
import chalk from 'chalk';
import debug from 'debug';
import glob from 'fast-glob';
import fs from 'fs-extra';
import webpack, { Configuration, Watching } from 'webpack';
import WebpackDevServer from 'webpack-dev-server';
Expand All @@ -25,6 +28,10 @@ const DEFAULT_LOGGER_PORT = 9000;
type WebpackToJsonOptions = Parameters<webpack.Stats['toJson']>[0];
type WebpackWatchHandler = Parameters<webpack.Compiler['watch']>[1];

type NativeDepsCtx = {
nativeDeps: Record<string, string[]>;
};

export default class WebpackPlugin extends PluginBase<WebpackPluginConfig> {
name = 'webpack';

Expand Down Expand Up @@ -160,61 +167,185 @@ export default class WebpackPlugin extends PluginBase<WebpackPluginConfig> {
await fs.remove(this.baseDir);

// TODO: Figure out how to get matrix from packager
let arches: string[] = [arch];
if (arch === 'universal') {
arches = ['arm64', 'x64'];
}
const arches: string[] = Array.from(
new Set(arch.split(',').reduce<string[]>((all, pArch) => (pArch === 'universal' ? all.concat(['arm64', 'x64']) : all.concat([pArch])), []))
);

return task.newListr(
arches.map(
(pArch): ForgeListrTaskDefinition => ({
title: `Building webpack bundle for ${chalk.magenta(pArch)}`,
task: async (_, task) => {
return task.newListr(
[
{
title: 'Preparing native dependencies',
task: async (_, innerTask) => {
await listrCompatibleRebuildHook(
this.projectDir,
await getElectronVersion(this.projectDir, await fs.readJson(path.join(this.projectDir, 'package.json'))),
platform,
pArch,
config.rebuildConfig,
innerTask
);
},
options: {
persistentOutput: true,
bottomBar: Infinity,
showTimer: true,
},
},
{
title: 'Building webpack bundles',
task: async () => {
await this.compileMain();
await this.compileRenderers();
// Store it in a place that won't get messed with
// We'll restore the right "arch" in the afterCopy hook further down
const targetDir = path.resolve(this.baseDir, pArch);
await fs.mkdirp(targetDir);
for (const child of await fs.readdir(this.baseDir)) {
if (!arches.includes(child)) {
await fs.move(path.resolve(this.baseDir, child), path.resolve(targetDir, child));
}
}
},
options: {
showTimer: true,
},
},
],
{ concurrent: false }
const firstArch = arches[0];
const otherArches = arches.slice(1);

const multiArchTasks: ListrTask<NativeDepsCtx>[] =
otherArches.length === 0
? []
: [
{
title: 'Mapping native dependencies',
task: async (ctx: NativeDepsCtx) => {
const firstArchDir = path.resolve(this.baseDir, firstArch);
const nodeModulesDir = path.resolve(this.projectDir, 'node_modules');
const mapping: Record<string, string[]> = Object.create(null);

const webpackNodeFiles = await glob('**/*.node', {
cwd: firstArchDir,
});
const nodeModulesNodeFiles = await glob('**/*.node', {
cwd: nodeModulesDir,
});
const hashToNodeModules: Record<string, string[]> = Object.create(null);

for (const nodeModulesNodeFile of nodeModulesNodeFiles) {
const hash = crypto.createHash('sha256');
const resolvedNodeFile = path.resolve(nodeModulesDir, nodeModulesNodeFile);
await pipeline(fs.createReadStream(resolvedNodeFile), hash);
const digest = hash.digest('hex');

hashToNodeModules[digest] = hashToNodeModules[digest] || [];
hashToNodeModules[digest].push(resolvedNodeFile);
}

for (const webpackNodeFile of webpackNodeFiles) {
const hash = crypto.createHash('sha256');
await pipeline(fs.createReadStream(path.resolve(firstArchDir, webpackNodeFile)), hash);
const matchedNodeModule = hashToNodeModules[hash.digest('hex')];
if (!matchedNodeModule || !matchedNodeModule.length) {
throw new Error(`Could not find originating native module for "${webpackNodeFile}"`);
}

mapping[webpackNodeFile] = matchedNodeModule;
}

ctx.nativeDeps = mapping;
},
},
{
title: `Generating multi-arch bundles`,
task: async (_, task) => {
return task.newListr(
otherArches.map(
(pArch): ListrTask<NativeDepsCtx> => ({
title: `Generating ${chalk.magenta(pArch)} bundle`,
task: async (_, innerTask) => {
return innerTask.newListr(
[
{
title: 'Preparing native dependencies',
task: async (_, innerTask) => {
await listrCompatibleRebuildHook(
this.projectDir,
await getElectronVersion(this.projectDir, await fs.readJson(path.join(this.projectDir, 'package.json'))),
platform,
pArch,
config.rebuildConfig,
innerTask
);
},
options: {
persistentOutput: true,
bottomBar: Infinity,
showTimer: true,
},
},
{
title: 'Mapping native dependencies',
task: async (ctx) => {
const nodeModulesDir = path.resolve(this.projectDir, 'node_modules');

// Duplicate the firstArch build
const firstDir = path.resolve(this.baseDir, firstArch);
const targetDir = path.resolve(this.baseDir, pArch);
await fs.mkdirp(targetDir);
for (const child of await fs.readdir(firstDir)) {
await fs.promises.cp(path.resolve(firstDir, child), path.resolve(targetDir, child), {
recursive: true,
});
}

const nodeModulesNodeFiles = await glob('**/*.node', {
cwd: nodeModulesDir,
});
const nodeModuleToHash: Record<string, string> = Object.create(null);

for (const nodeModulesNodeFile of nodeModulesNodeFiles) {
const hash = crypto.createHash('sha256');
const resolvedNodeFile = path.resolve(nodeModulesDir, nodeModulesNodeFile);
await pipeline(fs.createReadStream(resolvedNodeFile), hash);

nodeModuleToHash[resolvedNodeFile] = hash.digest('hex');
}

// Use the native module map to find the newly built native modules
for (const nativeDep of Object.keys(ctx.nativeDeps)) {
const archPath = path.resolve(targetDir, nativeDep);
await fs.remove(archPath);

const mappedPaths = ctx.nativeDeps[nativeDep];
if (!mappedPaths || !mappedPaths.length) {
throw new Error(`The "${nativeDep}" module could not be mapped to any native modules on disk`);
}

if (!mappedPaths.every((mappedPath) => nodeModuleToHash[mappedPath] === nodeModuleToHash[mappedPaths[0]])) {
throw new Error(
`The "${nativeDep}" mapped to multiple modules "${mappedPaths.join(
', '
)}" but the same modules post rebuild did not map to the same native code`
);
}

await fs.promises.cp(mappedPaths[0], archPath);
}
},
},
],
{ concurrent: false }
);
},
})
)
);
},
},
];

return task.newListr<NativeDepsCtx>(
[
{
title: `Preparing native dependencies for ${chalk.magenta(firstArch)}`,
task: async (_, innerTask) => {
await listrCompatibleRebuildHook(
this.projectDir,
await getElectronVersion(this.projectDir, await fs.readJson(path.join(this.projectDir, 'package.json'))),
platform,
firstArch,
config.rebuildConfig,
innerTask
);
},
})
),
options: {
persistentOutput: true,
bottomBar: Infinity,
showTimer: true,
},
},
{
title: 'Building webpack bundles',
task: async () => {
await this.compileMain();
await this.compileRenderers();
// Store it in a place that won't get messed with
// We'll restore the right "arch" in the afterCopy hook further down
const preExistingChildren = await fs.readdir(this.baseDir);
const targetDir = path.resolve(this.baseDir, firstArch);
await fs.mkdirp(targetDir);
for (const child of preExistingChildren) {
await fs.move(path.resolve(this.baseDir, child), path.resolve(targetDir, child));
}
},
options: {
showTimer: true,
},
},
...multiArchTasks,
],
{ concurrent: false }
// eslint-disable-next-line @typescript-eslint/no-explicit-any
) as any;
Expand Down
4 changes: 2 additions & 2 deletions packages/utils/core-utils/src/rebuild.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,13 @@ import * as path from 'path';
import { ForgeArch, ForgeListrTask, ForgePlatform } from '@electron-forge/shared-types';
import { RebuildOptions } from '@electron/rebuild';

export const listrCompatibleRebuildHook = async (
export const listrCompatibleRebuildHook = async <Ctx = never>(
buildPath: string,
electronVersion: string,
platform: ForgePlatform,
arch: ForgeArch,
config: Partial<RebuildOptions> = {},
task: ForgeListrTask<never>,
task: ForgeListrTask<Ctx>,
taskTitlePrefix = ''
): Promise<void> => {
task.title = `${taskTitlePrefix}Preparing native dependencies`;
Expand Down
1 change: 1 addition & 0 deletions packages/utils/types/src/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -204,6 +204,7 @@ export interface InitTemplateOptions {

// eslint-disable-next-line @typescript-eslint/no-explicit-any
export type ForgeListrTaskDefinition = ListrTask<never>;
export { ListrTask };

export interface ForgeTemplate {
requiredForgeVersion?: string;
Expand Down
6 changes: 3 additions & 3 deletions yarn.lock
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9679,9 +9679,9 @@ [email protected], mute-stream@~0.0.4:
integrity sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==

nan@^2.4.0:
version "2.15.0"
resolved "https://registry.yarnpkg.com/nan/-/nan-2.15.0.tgz#3f34a473ff18e15c1b5626b62903b5ad6e665fee"
integrity sha512-8ZtvEnA2c5aYCZYd1cvgdnU6cqwixRoYg70xPLWUws5ORTa/lnw+u4amixRS/Ac5U5mQVgp9pnlSUnbNWFaWZQ==
version "2.18.0"
resolved "https://registry.yarnpkg.com/nan/-/nan-2.18.0.tgz#26a6faae7ffbeb293a39660e88a76b82e30b7554"
integrity sha512-W7tfG7vMOGtD30sHoZSSc/JVYiyDPEyQVso/Zz+/uQd0B0L46gtC+pHha5FFMRpil6fm/AoEcRWyOVi4+E/f8w==

[email protected]:
version "3.3.1"
Expand Down

0 comments on commit 6e8e1ed

Please sign in to comment.