fix(HTTP Request Node): Use iconv-lite to decode http responses, to support more encoding types (#11930)

Author: netroy

Diff for: packages/@n8n/imap/package.json

@@ -20,7 +20,7 @@
     "dist/**/*"
   ],
   "dependencies": {
-    "iconv-lite": "0.6.3",
+    "iconv-lite": "catalog:",
     "imap": "0.8.19",
     "quoted-printable": "1.0.1",
     "utf8": "3.0.0",

Diff for: packages/core/package.json

@@ -47,6 +47,7 @@
     "fast-glob": "catalog:",
     "file-type": "16.5.4",
     "form-data": "catalog:",
+    "iconv-lite": "catalog:",
     "lodash": "catalog:",
     "luxon": "catalog:",
     "mime-types": "2.1.35",

Diff for: packages/core/src/NodeExecuteFunctions.ts

@@ -28,6 +28,7 @@ import { createReadStream } from 'fs';
 import { access as fsAccess, writeFile as fsWriteFile } from 'fs/promises';
 import { IncomingMessage } from 'http';
 import { Agent, type AgentOptions } from 'https';
+import iconv from 'iconv-lite';
 import get from 'lodash/get';
 import isEmpty from 'lodash/isEmpty';
 import merge from 'lodash/merge';
@@ -745,13 +746,13 @@ export function parseIncomingMessage(message: IncomingMessage) {
 	}
 }
 
-export async function binaryToString(body: Buffer | Readable, encoding?: BufferEncoding) {
-	const buffer = await binaryToBuffer(body);
+export async function binaryToString(body: Buffer | Readable, encoding?: string) {
 	if (!encoding && body instanceof IncomingMessage) {
 		parseIncomingMessage(body);
 		encoding = body.encoding;
 	}
-	return buffer.toString(encoding);
+	const buffer = await binaryToBuffer(body);
+	return iconv.decode(buffer, encoding ?? 'utf-8');
 }
 
 export async function proxyRequestToAxios(

Diff for: packages/core/test/NodeExecuteFunctions.test.ts

@@ -1,5 +1,5 @@
 import { mkdtempSync, readFileSync } from 'fs';
-import type { IncomingMessage } from 'http';
+import { IncomingMessage } from 'http';
 import type { Agent } from 'https';
 import { mock } from 'jest-mock-extended';
 import type {
@@ -16,12 +16,14 @@ import type {
 import nock from 'nock';
 import { tmpdir } from 'os';
 import { join } from 'path';
+import { Readable } from 'stream';
 import type { SecureContextOptions } from 'tls';
 import Container from 'typedi';
 
 import { BinaryDataService } from '@/BinaryData/BinaryData.service';
 import { InstanceSettings } from '@/InstanceSettings';
 import {
+	binaryToString,
 	copyInputItems,
 	getBinaryDataBuffer,
 	isFilePathBlocked,
@@ -549,6 +551,101 @@ describe('NodeExecuteFunctions', () => {
 			},
 		);
 	});
+
+	describe('binaryToString', () => {
+		const ENCODING_SAMPLES = {
+			utf8: {
+				text: 'Hello, 世界! τεστ мир ⚡️ é à ü ñ',
+				buffer: Buffer.from([
+					0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x2c, 0x20, 0xe4, 0xb8, 0x96, 0xe7, 0x95, 0x8c, 0x21, 0x20,
+					0xcf, 0x84, 0xce, 0xb5, 0xcf, 0x83, 0xcf, 0x84, 0x20, 0xd0, 0xbc, 0xd0, 0xb8, 0xd1, 0x80,
+					0x20, 0xe2, 0x9a, 0xa1, 0xef, 0xb8, 0x8f, 0x20, 0xc3, 0xa9, 0x20, 0xc3, 0xa0, 0x20, 0xc3,
+					0xbc, 0x20, 0xc3, 0xb1,
+				]),
+			},
+
+			'iso-8859-15': {
+				text: 'Café € personnalité',
+				buffer: Buffer.from([
+					0x43, 0x61, 0x66, 0xe9, 0x20, 0xa4, 0x20, 0x70, 0x65, 0x72, 0x73, 0x6f, 0x6e, 0x6e, 0x61,
+					0x6c, 0x69, 0x74, 0xe9,
+				]),
+			},
+
+			latin1: {
+				text: 'señor année déjà',
+				buffer: Buffer.from([
+					0x73, 0x65, 0xf1, 0x6f, 0x72, 0x20, 0x61, 0x6e, 0x6e, 0xe9, 0x65, 0x20, 0x64, 0xe9, 0x6a,
+					0xe0,
+				]),
+			},
+
+			ascii: {
+				text: 'Hello, World! 123',
+				buffer: Buffer.from([
+					0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x2c, 0x20, 0x57, 0x6f, 0x72, 0x6c, 0x64, 0x21, 0x20, 0x31,
+					0x32, 0x33,
+				]),
+			},
+
+			'windows-1252': {
+				text: '€ Smart "quotes" • bullet',
+				buffer: Buffer.from([
+					0x80, 0x20, 0x53, 0x6d, 0x61, 0x72, 0x74, 0x20, 0x22, 0x71, 0x75, 0x6f, 0x74, 0x65, 0x73,
+					0x22, 0x20, 0x95, 0x20, 0x62, 0x75, 0x6c, 0x6c, 0x65, 0x74,
+				]),
+			},
+
+			'shift-jis': {
+				text: 'こんにちは世界',
+				buffer: Buffer.from([
+					0x82, 0xb1, 0x82, 0xf1, 0x82, 0xc9, 0x82, 0xbf, 0x82, 0xcd, 0x90, 0xa2, 0x8a, 0x45,
+				]),
+			},
+
+			big5: {
+				text: '哈囉世界',
+				buffer: Buffer.from([0xab, 0xa2, 0xc5, 0x6f, 0xa5, 0x40, 0xac, 0xc9]),
+			},
+
+			'koi8-r': {
+				text: 'Привет мир',
+				buffer: Buffer.from([0xf0, 0xd2, 0xc9, 0xd7, 0xc5, 0xd4, 0x20, 0xcd, 0xc9, 0xd2]),
+			},
+		};
+
+		describe('should handle Buffer', () => {
+			for (const [encoding, { text, buffer }] of Object.entries(ENCODING_SAMPLES)) {
+				test(`with ${encoding}`, async () => {
+					const data = await binaryToString(buffer, encoding);
+					expect(data).toBe(text);
+				});
+			}
+		});
+
+		describe('should handle streams', () => {
+			for (const [encoding, { text, buffer }] of Object.entries(ENCODING_SAMPLES)) {
+				test(`with ${encoding}`, async () => {
+					const stream = Readable.from(buffer);
+					const data = await binaryToString(stream, encoding);
+					expect(data).toBe(text);
+				});
+			}
+		});
+
+		describe('should handle IncomingMessage', () => {
+			for (const [encoding, { text, buffer }] of Object.entries(ENCODING_SAMPLES)) {
+				test(`with ${encoding}`, async () => {
+					const response = Readable.from(buffer) as IncomingMessage;
+					response.headers = { 'content-type': `application/json;charset=${encoding}` };
+					// @ts-expect-error need this hack to fake `instanceof IncomingMessage` checks
+					response.__proto__ = IncomingMessage.prototype;
+					const data = await binaryToString(response);
+					expect(data).toBe(text);
+				});
+			}
+		});
+	});
 });
 
 describe('isFilePathBlocked', () => {

Diff for: packages/nodes-base/package.json

@@ -868,7 +868,7 @@
     "get-system-fonts": "2.0.2",
     "gm": "1.25.0",
     "html-to-text": "9.0.5",
-    "iconv-lite": "0.6.3",
+    "iconv-lite": "catalog:",
     "ics": "2.40.0",
     "isbot": "3.6.13",
     "iso-639-1": "2.1.15",

Diff for: pnpm-lock.yaml

@@ -18,6 +18,7 @@ catalog:
   fast-glob: 3.2.12
   flatted: 3.2.7
   form-data: 4.0.0
+  iconv-lite: 0.6.3
   lodash: 4.17.21
   luxon: 3.4.4
   nanoid: 3.3.6