Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor: only run webpack once for multi-arch packages #3437

Merged
merged 2 commits into from
Dec 6, 2023
Merged

refactor: only run webpack once for multi-arch packages #3437

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7ec0377
refactor: only run webpack once for multi-arch packages
MarshallOfSound Dec 4, 2023
cecad94
fix: handle comma seperated arch string
MarshallOfSound Dec 4, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion packages/plugin/base/src/Plugin.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ export default abstract class Plugin<C> implements IForgePlugin {
// This method is not type safe internally, but is type safe for consumers
// @internal
export const namedHookWithTaskFn = <Hook extends ForgeHookName>(
hookFn: (task: ForgeListrTask<never> | null, ...args: Parameters<ForgeHookFn<Hook>>) => ReturnType<ForgeHookFn<Hook>>,
hookFn: <Ctx = never>(task: ForgeListrTask<Ctx> | null, ...args: Parameters<ForgeHookFn<Hook>>) => ReturnType<ForgeHookFn<Hook>>,
name: string
): ForgeHookFn<Hook> => {
function namedHookWithTaskInner(this: ForgeListrTask<any> | null, ...args: any[]) {
Expand Down
1 change: 1 addition & 0 deletions packages/plugin/webpack/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
"@electron-forge/web-multi-logger": "7.2.0",
"chalk": "^4.0.0",
"debug": "^4.3.1",
"fast-glob": "^3.2.7",
"fs-extra": "^10.0.0",
"html-webpack-plugin": "^5.5.3",
"webpack": "^5.69.1",
Expand Down
237 changes: 184 additions & 53 deletions packages/plugin/webpack/src/WebpackPlugin.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,15 @@
import crypto from 'crypto';
import http from 'http';
import path from 'path';
import { pipeline } from 'stream/promises';

import { getElectronVersion, listrCompatibleRebuildHook } from '@electron-forge/core-utils';
import { namedHookWithTaskFn, PluginBase } from '@electron-forge/plugin-base';
import { ForgeListrTaskDefinition, ForgeMultiHookMap, ResolvedForgeConfig, StartResult } from '@electron-forge/shared-types';
import { ForgeMultiHookMap, ListrTask, ResolvedForgeConfig, StartResult } from '@electron-forge/shared-types';
import Logger, { Tab } from '@electron-forge/web-multi-logger';
import chalk from 'chalk';
import debug from 'debug';
import glob from 'fast-glob';
import fs from 'fs-extra';
import webpack, { Configuration, Watching } from 'webpack';
import WebpackDevServer from 'webpack-dev-server';
Expand All @@ -25,6 +28,10 @@ const DEFAULT_LOGGER_PORT = 9000;
type WebpackToJsonOptions = Parameters<webpack.Stats['toJson']>[0];
type WebpackWatchHandler = Parameters<webpack.Compiler['watch']>[1];

type NativeDepsCtx = {
nativeDeps: Record<string, string[]>;
};

export default class WebpackPlugin extends PluginBase<WebpackPluginConfig> {
name = 'webpack';

Expand Down Expand Up @@ -160,61 +167,185 @@ export default class WebpackPlugin extends PluginBase<WebpackPluginConfig> {
await fs.remove(this.baseDir);

// TODO: Figure out how to get matrix from packager
let arches: string[] = [arch];
if (arch === 'universal') {
arches = ['arm64', 'x64'];
}
const arches: string[] = Array.from(
new Set(arch.split(',').reduce<string[]>((all, pArch) => (pArch === 'universal' ? all.concat(['arm64', 'x64']) : all.concat([pArch])), []))
);

return task.newListr(
arches.map(
(pArch): ForgeListrTaskDefinition => ({
title: `Building webpack bundle for ${chalk.magenta(pArch)}`,
task: async (_, task) => {
return task.newListr(
[
{
title: 'Preparing native dependencies',
task: async (_, innerTask) => {
await listrCompatibleRebuildHook(
this.projectDir,
await getElectronVersion(this.projectDir, await fs.readJson(path.join(this.projectDir, 'package.json'))),
platform,
pArch,
config.rebuildConfig,
innerTask
);
},
options: {
persistentOutput: true,
bottomBar: Infinity,
showTimer: true,
},
},
{
title: 'Building webpack bundles',
task: async () => {
await this.compileMain();
await this.compileRenderers();
// Store it in a place that won't get messed with
// We'll restore the right "arch" in the afterCopy hook further down
const targetDir = path.resolve(this.baseDir, pArch);
await fs.mkdirp(targetDir);
for (const child of await fs.readdir(this.baseDir)) {
if (!arches.includes(child)) {
await fs.move(path.resolve(this.baseDir, child), path.resolve(targetDir, child));
}
}
},
options: {
showTimer: true,
},
},
],
{ concurrent: false }
const firstArch = arches[0];
const otherArches = arches.slice(1);

const multiArchTasks: ListrTask<NativeDepsCtx>[] =
otherArches.length === 0
? []
: [
{
title: 'Mapping native dependencies',
task: async (ctx: NativeDepsCtx) => {
const firstArchDir = path.resolve(this.baseDir, firstArch);
const nodeModulesDir = path.resolve(this.projectDir, 'node_modules');
const mapping: Record<string, string[]> = Object.create(null);

const webpackNodeFiles = await glob('**/*.node', {
cwd: firstArchDir,
});
const nodeModulesNodeFiles = await glob('**/*.node', {
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In retrospect, this line probably caused #3526.

cwd: nodeModulesDir,
});
const hashToNodeModules: Record<string, string[]> = Object.create(null);

for (const nodeModulesNodeFile of nodeModulesNodeFiles) {
const hash = crypto.createHash('sha256');
const resolvedNodeFile = path.resolve(nodeModulesDir, nodeModulesNodeFile);
await pipeline(fs.createReadStream(resolvedNodeFile), hash);
const digest = hash.digest('hex');

hashToNodeModules[digest] = hashToNodeModules[digest] || [];
hashToNodeModules[digest].push(resolvedNodeFile);
}

for (const webpackNodeFile of webpackNodeFiles) {
const hash = crypto.createHash('sha256');
await pipeline(fs.createReadStream(path.resolve(firstArchDir, webpackNodeFile)), hash);
const matchedNodeModule = hashToNodeModules[hash.digest('hex')];
if (!matchedNodeModule || !matchedNodeModule.length) {
throw new Error(`Could not find originating native module for "${webpackNodeFile}"`);
}

mapping[webpackNodeFile] = matchedNodeModule;
}

ctx.nativeDeps = mapping;
},
},
{
title: `Generating multi-arch bundles`,
task: async (_, task) => {
return task.newListr(
otherArches.map(
(pArch): ListrTask<NativeDepsCtx> => ({
title: `Generating ${chalk.magenta(pArch)} bundle`,
task: async (_, innerTask) => {
return innerTask.newListr(
[
{
title: 'Preparing native dependencies',
task: async (_, innerTask) => {
await listrCompatibleRebuildHook(
this.projectDir,
await getElectronVersion(this.projectDir, await fs.readJson(path.join(this.projectDir, 'package.json'))),
platform,
pArch,
config.rebuildConfig,
innerTask
);
},
options: {
persistentOutput: true,
bottomBar: Infinity,
showTimer: true,
},
},
{
title: 'Mapping native dependencies',
task: async (ctx) => {
const nodeModulesDir = path.resolve(this.projectDir, 'node_modules');

// Duplicate the firstArch build
const firstDir = path.resolve(this.baseDir, firstArch);
const targetDir = path.resolve(this.baseDir, pArch);
await fs.mkdirp(targetDir);
for (const child of await fs.readdir(firstDir)) {
await fs.promises.cp(path.resolve(firstDir, child), path.resolve(targetDir, child), {
recursive: true,
});
}

const nodeModulesNodeFiles = await glob('**/*.node', {
cwd: nodeModulesDir,
});
const nodeModuleToHash: Record<string, string> = Object.create(null);

for (const nodeModulesNodeFile of nodeModulesNodeFiles) {
const hash = crypto.createHash('sha256');
const resolvedNodeFile = path.resolve(nodeModulesDir, nodeModulesNodeFile);
await pipeline(fs.createReadStream(resolvedNodeFile), hash);

nodeModuleToHash[resolvedNodeFile] = hash.digest('hex');
}

// Use the native module map to find the newly built native modules
for (const nativeDep of Object.keys(ctx.nativeDeps)) {
const archPath = path.resolve(targetDir, nativeDep);
await fs.remove(archPath);

const mappedPaths = ctx.nativeDeps[nativeDep];
if (!mappedPaths || !mappedPaths.length) {
throw new Error(`The "${nativeDep}" module could not be mapped to any native modules on disk`);
}

if (!mappedPaths.every((mappedPath) => nodeModuleToHash[mappedPath] === nodeModuleToHash[mappedPaths[0]])) {
throw new Error(
`The "${nativeDep}" mapped to multiple modules "${mappedPaths.join(
', '
)}" but the same modules post rebuild did not map to the same native code`
);
}

await fs.promises.cp(mappedPaths[0], archPath);
}
},
},
],
{ concurrent: false }
);
},
})
)
);
},
},
];

return task.newListr<NativeDepsCtx>(
[
{
title: `Preparing native dependencies for ${chalk.magenta(firstArch)}`,
task: async (_, innerTask) => {
await listrCompatibleRebuildHook(
this.projectDir,
await getElectronVersion(this.projectDir, await fs.readJson(path.join(this.projectDir, 'package.json'))),
platform,
firstArch,
config.rebuildConfig,
innerTask
);
},
})
),
options: {
persistentOutput: true,
bottomBar: Infinity,
showTimer: true,
},
},
{
title: 'Building webpack bundles',
task: async () => {
await this.compileMain();
await this.compileRenderers();
// Store it in a place that won't get messed with
// We'll restore the right "arch" in the afterCopy hook further down
const preExistingChildren = await fs.readdir(this.baseDir);
const targetDir = path.resolve(this.baseDir, firstArch);
await fs.mkdirp(targetDir);
for (const child of preExistingChildren) {
await fs.move(path.resolve(this.baseDir, child), path.resolve(targetDir, child));
}
},
options: {
showTimer: true,
},
},
...multiArchTasks,
],
{ concurrent: false }
// eslint-disable-next-line @typescript-eslint/no-explicit-any
) as any;
Expand Down
4 changes: 2 additions & 2 deletions packages/utils/core-utils/src/rebuild.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,13 @@ import * as path from 'path';
import { ForgeArch, ForgeListrTask, ForgePlatform } from '@electron-forge/shared-types';
import { RebuildOptions } from '@electron/rebuild';

export const listrCompatibleRebuildHook = async (
export const listrCompatibleRebuildHook = async <Ctx = never>(
buildPath: string,
electronVersion: string,
platform: ForgePlatform,
arch: ForgeArch,
config: Partial<RebuildOptions> = {},
task: ForgeListrTask<never>,
task: ForgeListrTask<Ctx>,
taskTitlePrefix = ''
): Promise<void> => {
task.title = `${taskTitlePrefix}Preparing native dependencies`;
Expand Down
1 change: 1 addition & 0 deletions packages/utils/types/src/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -204,6 +204,7 @@ export interface InitTemplateOptions {

// eslint-disable-next-line @typescript-eslint/no-explicit-any
export type ForgeListrTaskDefinition = ListrTask<never>;
export { ListrTask };

export interface ForgeTemplate {
requiredForgeVersion?: string;
Expand Down
6 changes: 3 additions & 3 deletions yarn.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9679,9 +9679,9 @@ [email protected], mute-stream@~0.0.4:
integrity sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==

nan@^2.4.0:
version "2.15.0"
resolved "https://registry.yarnpkg.com/nan/-/nan-2.15.0.tgz#3f34a473ff18e15c1b5626b62903b5ad6e665fee"
integrity sha512-8ZtvEnA2c5aYCZYd1cvgdnU6cqwixRoYg70xPLWUws5ORTa/lnw+u4amixRS/Ac5U5mQVgp9pnlSUnbNWFaWZQ==
version "2.18.0"
resolved "https://registry.yarnpkg.com/nan/-/nan-2.18.0.tgz#26a6faae7ffbeb293a39660e88a76b82e30b7554"
integrity sha512-W7tfG7vMOGtD30sHoZSSc/JVYiyDPEyQVso/Zz+/uQd0B0L46gtC+pHha5FFMRpil6fm/AoEcRWyOVi4+E/f8w==

[email protected]:
version "3.3.1"
Expand Down