✨ Make possible to dynamically install nodes from npm

packages/cli/package.json

@@ -70,7 +70,6 @@
     "@types/node": "14.17.27",
     "@types/open": "^6.1.0",
     "@types/parseurl": "^1.3.1",
-    "@types/tar": "~6.1.1",
     "@types/request-promise-native": "~1.0.15",
     "@types/validator": "^13.7.0",
     "axios": "^0.21.1",
@@ -126,7 +125,6 @@
     "request-promise-native": "^1.0.7",
     "sqlite3": "^5.0.1",
     "sse-channel": "^3.1.1",
-    "tar": "~6.1.11",
     "tslib": "1.14.1",
     "typeorm": "0.2.30",
     "winston": "^3.3.3"

packages/cli/src/LoadNodesAndCredentials.ts

@@ -28,19 +28,18 @@ import {
 	mkdir as fsMkdir,
 	readdir as fsReaddir,
 	readFile as fsReadFile,
-	rename as fsRename,
-	rm as fsRm,
 	stat as fsStat,
-	writeFile as fsWriteFile,
 } from 'fs/promises';
 import * as glob from 'fast-glob';
 import * as path from 'path';
-import * as requestPromise from 'request-promise-native';
-import { extract } from 'tar';
+import { exec } from 'child_process';
+import { promisify } from 'util';
 import { IN8nNodePackageJson } from './Interfaces';
 import { getLogger } from './Logger';
 import * as config from '../config';
 
+const execAsync = promisify(exec);
+
 const CUSTOM_NODES_CATEGORY = 'Custom Nodes';
 
 class LoadNodesAndCredentialsClass {
@@ -95,24 +94,21 @@ class LoadNodesAndCredentialsClass {
 		this.includeNodes = config.get('nodes.include');
 
 		// Get all the installed packages which contain n8n nodes
-		const packages = await this.getN8nNodePackages();
-
-		for (const packageName of packages) {
-			await this.loadDataFromPackage(path.join(this.nodeModulesPath, packageName));
-		}
-
-		// Read downloaded nodes and credentials
-		const downloadedNodesFolder = UserSettings.getUserN8nFolderDowloadedNodesPath();
-		for (const folderContent of await fsReaddir(downloadedNodesFolder)) {
-			if (!(await fsStat(path.join(downloadedNodesFolder, folderContent))).isDirectory()) {
-				continue;
-			}
-
-			const packagePath = path.join(downloadedNodesFolder, folderContent, 'package');
-			if (!(await fsStat(packagePath)).isDirectory()) {
-				continue;
-			}
+		let nodePackages = await this.getN8nNodePackages(this.nodeModulesPath);
 
+		try {
+			// Read downloaded nodes and credentials
+			const downloadedNodesFolder = UserSettings.getUserN8nFolderDowloadedNodesPath();
+			const downloadedNodesFolderModules = path.join(downloadedNodesFolder, 'node_modules');
+			await fsAccess(downloadedNodesFolderModules);
+			nodePackages = [
+				...nodePackages,
+				...(await this.getN8nNodePackages(downloadedNodesFolderModules)),
+			];
+			// eslint-disable-next-line no-empty
+		} catch (error) {}
+
+		for (const packagePath of nodePackages) {
 			await this.loadDataFromPackage(packagePath);
 		}
 
@@ -142,10 +138,10 @@ class LoadNodesAndCredentialsClass {
 	 * @returns {Promise<string[]>}
 	 * @memberof LoadNodesAndCredentialsClass
 	 */
-	async getN8nNodePackages(): Promise<string[]> {
+	async getN8nNodePackages(baseModulesPath: string): Promise<string[]> {
 		const getN8nNodePackagesRecursive = async (relativePath: string): Promise<string[]> => {
 			const results: string[] = [];
-			const nodeModulesPath = `${this.nodeModulesPath}/${relativePath}`;
+			const nodeModulesPath = `${baseModulesPath}/${relativePath}`;
 			for (const file of await fsReaddir(nodeModulesPath)) {
 				const isN8nNodesPackage = file.indexOf('n8n-nodes-') === 0;
 				const isNpmScopedPackage = file.indexOf('@') === 0;
@@ -156,7 +152,7 @@ class LoadNodesAndCredentialsClass {
 					continue;
 				}
 				if (isN8nNodesPackage) {
-					results.push(`${relativePath}${file}`);
+					results.push(`${baseModulesPath}/${relativePath}${file}`);
 				}
 				if (isNpmScopedPackage) {
 					results.push(...(await getN8nNodePackagesRecursive(`${relativePath}${file}/`)));
@@ -205,45 +201,38 @@ class LoadNodesAndCredentialsClass {
 		};
 	}
 
-	async loadNpmModuleFromUrl(url: string): Promise<INodeTypeNameVersion[]> {
-		let data: Uint8Array;
+	async loadNpmModule(packageName: string): Promise<INodeTypeNameVersion[]> {
+		const downloadFolder = UserSettings.getUserN8nFolderDowloadedNodesPath();
+
+		// Make sure the node-download folder exists
 		try {
-			data = await requestPromise.get(url, { encoding: null });
+			await fsAccess(downloadFolder);
+			// eslint-disable-next-line no-empty
 		} catch (error) {
-			throw new Error('Could not download node package.');
+			await fsMkdir(downloadFolder);
 		}
 
-		const urlParts = path.parse(url);
-
-		const downloadFolder = UserSettings.getUserN8nFolderDowloadedNodesPath();
-		const tempNodeUnpackedPath = path.join(downloadFolder, urlParts.name);
-		const nodeTarFilePath = tempNodeUnpackedPath + urlParts.ext;
-
-		await fsMkdir(tempNodeUnpackedPath);
-		await fsWriteFile(nodeTarFilePath, data, 'binary');
-		await extract({
-			file: nodeTarFilePath,
-			cwd: tempNodeUnpackedPath,
-		});
-
-		// Delete the node tar file
-		await fsRm(nodeTarFilePath);
-
-		const packagePath = path.join(tempNodeUnpackedPath, 'package');
+		const command = `npm install ${packageName}`;
+		const execOptions = {
+			cwd: downloadFolder,
+			env: {
+				NODE_PATH: process.env.NODE_PATH,
+				PATH: process.env.PATH,
+			},
+		};
 
-		// Get the information from the package.json file
-		const packageFile = await this.readPackageJson(packagePath);
+		try {
+			await execAsync(command, execOptions);
+		} catch (error) {
+			if (error.message.includes('404 Not Found')) {
+				throw new Error(`The npm package "${packageName}" could not be found.`);
+			}
+			throw error;
+		}
 
-		// Fix package path
-		const finalNodeUnpackedPath = path.join(
-			downloadFolder,
-			// Replace slash for packages that have namespace
-			packageFile.name.replace(new RegExp('/'), '_'),
-		);
-		await fsRename(tempNodeUnpackedPath, finalNodeUnpackedPath);
+		const finalNodeUnpackedPath = path.join(downloadFolder, 'node_modules', packageName);
 
-		// Load credentials and nodes
-		return this.loadDataFromPackage(path.join(finalNodeUnpackedPath, 'package'));
+		return this.loadDataFromPackage(finalNodeUnpackedPath);
 	}
 
 	/**

packages/cli/src/Server.ts

@@ -613,13 +613,17 @@ class App {
 		this.app.post(
 			`/${this.restEndpoint}/node`,
 			ResponseHelper.send(async (req: express.Request, res: express.Response) => {
-				const url = req.body.url as string;
-				if (url === undefined) {
-					throw new ResponseHelper.ResponseError(`The parameter "url" is missing!`, undefined, 400);
+				const name = req.body.name as string;
+				if (name === undefined) {
+					throw new ResponseHelper.ResponseError(
+						`The parameter "name" is missing!`,
+						undefined,
+						400,
+					);
 				}
 
 				try {
-					const nodes = await LoadNodesAndCredentials().loadNpmModuleFromUrl(url);
+					const nodes = await LoadNodesAndCredentials().loadNpmModule(name);
 
 					// Inform the connected frontends that new nodes are available
 					nodes.forEach((nodeData) => {
@@ -632,7 +636,7 @@ class App {
 					};
 				} catch (error) {
 					throw new ResponseHelper.ResponseError(
-						`Error loading nodes from "${url}": ${error.message}`,
+						`Error loading package "${name}": ${error.message}`,
 						undefined,
 						500,
 					);

packages/core/src/UserSettings.ts

@@ -277,7 +277,6 @@ export function getUserN8nFolderCustomExtensionPath(): string {
  * @returns {string}
  */
 export function getUserN8nFolderDowloadedNodesPath(): string {
-	// TODO: Replace "nodes" with constant
 	return path.join(getUserN8nFolderPath(), DOWNLOADED_NODES_SUBDIRECTORY);
 }