feat(rpc): make Buffer dependency optional

fix http config issue

packages/bson/src/bson-serializer.ts

@@ -8,7 +8,15 @@
  * You should have received a copy of the MIT License along with this program.
  */
 
-import { CompilerContext, hasProperty, isArray, isIterable, isObject, toFastProperties } from '@deepkit/core';
+import {
+    CompilerContext,
+    createBuffer,
+    hasProperty,
+    isArray,
+    isIterable,
+    isObject,
+    toFastProperties,
+} from '@deepkit/core';
 import {
     binaryBigIntAnnotation,
     BinaryBigIntType,
@@ -101,10 +109,6 @@ import {
     TWO_PWR_32_DBL_N,
 } from './utils.js';
 
-export function createBuffer(size: number): Uint8Array {
-    return 'undefined' !== typeof Buffer && 'function' === typeof Buffer.allocUnsafe ? Buffer.allocUnsafe(size) : new Uint8Array(size);
-}
-
 // BSON MAX VALUES
 const BSON_INT32_MAX = 0x7fffffff;
 const BSON_INT32_MIN = -0x80000000;

packages/core/index.ts

@@ -24,6 +24,7 @@ export * from './src/reflection.js';
 export * from './src/url.js';
 export * from './src/array.js';
 export * from './src/types.js';
+export * from './src/buffer.js';
 
 // export * does not work for some reason
 // we get: packages/framework/src/debug/media.controller.ts:14:25 - error TS2305: Module '"@deepkit/core"' has no exported member 'pathJoin'.

packages/core/src/buffer.ts

@@ -0,0 +1,56 @@
+declare const Buffer: any;
+
+/**
+ * Create a buffer of the given size (uninitialized, so it may contain random data).
+ *
+ * Note that the result is either Uin8Array or Buffer, depending on the environment.
+ * Buffer from NodeJS works slightly different than Uint8Array.
+ */
+export const createBuffer: (size: number) => Uint8Array = 'undefined' !== typeof Buffer && 'function' === typeof Buffer.allocUnsafe ? Buffer.allocUnsafe : (size) => new Uint8Array(size);
+
+/**
+ * Concat multiple buffers into one.
+ */
+export function bufferConcat(chunks: Uint8Array[], length?: number): Uint8Array {
+    if (undefined === length) {
+        length = 0;
+        for (const chunk of chunks) length += chunk.length;
+    }
+
+    const result = createBuffer(length);
+    let offset = 0;
+    for (const chunk of chunks) {
+        result.set(chunk, offset);
+        offset += chunk.length;
+    }
+
+    return result;
+}
+
+const textEncoder = new TextDecoder();
+
+export const uint8ArrayToUtf8 = 'undefined' !== typeof Buffer ? (buffer: Uint8Array) => Buffer.from(buffer).toString('utf8') : (buffer: Uint8Array) => textEncoder.decode(buffer);
+
+/**
+ * Convert a buffer to a string.
+ */
+export function bufferToString(buffer: string | Uint8Array): string {
+    if ('string' === typeof buffer) return buffer;
+    return uint8ArrayToUtf8(buffer);
+}
+
+export function nativeBase64ToUint8Array(base64: string): Uint8Array {
+    const raw = atob(base64);
+    const rawLength = raw.length;
+    const array = createBuffer(rawLength);
+
+    for (let i = 0; i < rawLength; i++) {
+        array[i] = raw.charCodeAt(i);
+    }
+    return array;
+}
+
+/**
+ * Converts a base64 string to a Uint8Array.
+ */
+export const base64ToUint8Array: (v: string) => Uint8Array = 'undefined' === typeof Buffer ? nativeBase64ToUint8Array : (base64: string) => Buffer.from(base64, 'base64');

packages/framework/src/module.config.ts

@@ -7,7 +7,7 @@
  *
  * You should have received a copy of the MIT License along with this program.
  */
-import { HttpConfig, HttpParserOptions } from '@deepkit/http';
+import { HttpConfig } from '@deepkit/http';
 
 const isWindows = 'undefined' !== typeof process ? process.platform === 'win32' : false;
 
@@ -140,8 +140,6 @@ export class FrameworkConfig {
      */
     httpLog: boolean = true;
 
-    httpParse: HttpParserOptions = {};
-
     /**
      * @description The session ClassType
      */

packages/framework/src/module.ts

@@ -191,7 +191,7 @@ export class FrameworkModule extends createModule({
             this.addListener(HttpLogger);
         }
 
-        this.getImportedModuleByClass(HttpModule).configure({ ...this.config.http, parser: this.config.httpParse });
+        this.getImportedModuleByClass(HttpModule).configure(this.config.http);
 
         if (this.config.publicDir) {
             const localPublicDir = join(process.cwd(), this.config.publicDir);

packages/framework/tests/http.spec.ts

@@ -1,4 +1,3 @@
-import { App } from '@deepkit/app';
 import { FrameworkModule } from '../src/module.js';
 import { expect, test } from '@jest/globals';
 import { HttpBody, HttpKernel, HttpRequest, HttpRouterRegistry } from '@deepkit/http';
@@ -25,8 +24,10 @@ test('functional http app', async () => {
 test('http parse config', async () => {
     const test = createTestingApp({
         imports: [new FrameworkModule({
-            httpParse: {
-                maxFields: 1
+            http: {
+                parser: {
+                    maxFields: 1
+                }
             }
         })]
     });

packages/mongo/src/client/command/auth/scram.ts

@@ -15,6 +15,7 @@ import { BaseResponse, Command } from '../command.js';
 import { MongoError } from '../../error.js';
 // @ts-ignore
 import saslprep from 'saslprep';
+import { base64ToUint8Array } from '@deepkit/core';
 
 interface SaslStartCommand {
     saslStart: 1;
@@ -66,7 +67,7 @@ function passwordDigest(u: string, p: string) {
     return md5.digest('hex');
 }
 
-function HI(data: string, salt: Buffer, iterations: number, cryptoAlgorithm: string) {
+function HI(data: string, salt: Uint8Array, iterations: number, cryptoAlgorithm: string) {
     if (cryptoAlgorithm !== 'sha1' && cryptoAlgorithm !== 'sha256') {
         throw new MongoError(`Invalid crypto algorithm ${cryptoAlgorithm}`);
     }
@@ -116,7 +117,7 @@ export abstract class ScramAuth implements MongoAuth {
         const withoutProof = `c=biws,r=${payloadStart.r}`;
         const saltedPassword = HI(
             processedPassword,
-            Buffer.from(payloadStart.s, 'base64'),
+            base64ToUint8Array(payloadStart.s),
             payloadStart.i,
             this.cryptoMethod
         );

packages/rpc/src/model.ts

@@ -8,7 +8,7 @@
  * You should have received a copy of the MIT License along with this program.
  */
 
-import { ClassType, isObject } from '@deepkit/core';
+import { bufferConcat, ClassType, isObject } from '@deepkit/core';
 import { tearDown } from '@deepkit/core-rxjs';
 import { arrayBufferTo, entity } from '@deepkit/type';
 import { BehaviorSubject, Observable, Subject, TeardownLogic } from 'rxjs';
@@ -121,7 +121,7 @@ export class StreamBehaviorSubject<T> extends BehaviorSubject<T> {
         if (this.nextOnAppend) {
             if (value instanceof Uint8Array) {
                 if (this.value instanceof Uint8Array) {
-                    this.next(Buffer.concat([this.value as any, value as any]) as any);
+                    this.next(bufferConcat([this.value as any, value as any]) as any);
                 } else {
                     this.next(value as any);
                 }

packages/rpc/src/protocol.ts

@@ -15,7 +15,7 @@ import {
     getBSONSizer,
     Writer,
 } from '@deepkit/bson';
-import { ClassType } from '@deepkit/core';
+import { bufferConcat, ClassType, createBuffer } from '@deepkit/core';
 import { rpcChunk, rpcError, RpcTypes } from './model.js';
 import type { SingleProgress } from './progress.js';
 import {
@@ -234,9 +234,6 @@ export function readBinaryRpcMessage(buffer: Uint8Array): RpcMessage {
 
     return new RpcMessage(id, composite, type, routeType, offset, size - offset, buffer);
 }
-
-export const createBuffer: (size: number) => Uint8Array = 'undefined' !== typeof Buffer && 'function' === typeof Buffer.allocUnsafe ? Buffer.allocUnsafe : (size) => new Uint8Array(size);
-
 export interface RpcCreateMessageDef<T> {
     type: number;
     schema?: Type;
@@ -593,12 +590,7 @@ export class RpcBinaryMessageReader {
                 this.progress.delete(message.id);
                 this.chunks.delete(body.id);
                 this.chunkAcks.delete(body.id);
-                let offset = 0;
-                const newBuffer = createBuffer(body.total);
-                for (const buffer of chunks.buffers) {
-                    newBuffer.set(buffer, offset);
-                    offset += buffer.byteLength;
-                }
+                const newBuffer = bufferConcat(chunks.buffers, body.total);
                 const packedMessage = readBinaryRpcMessage(newBuffer);
                 this.onMessage(packedMessage);
             }
@@ -642,7 +634,7 @@ export class RpcBinaryBufferReader {
             this.currentMessage = data.byteLength === bytes ? data : data.slice(0, bytes);
             this.currentMessageSize = readUint32LE(data);
         } else {
-            this.currentMessage = Buffer.concat([this.currentMessage, data.byteLength === bytes ? data : data.slice(0, bytes)]);
+            this.currentMessage = bufferConcat([this.currentMessage, data.byteLength === bytes ? data : data.slice(0, bytes)]);
             if (!this.currentMessageSize) {
                 if (this.currentMessage.byteLength < 4) {
                     //not enough data to read the header. Wait for next onData

packages/rpc/src/server/http.ts

@@ -1,5 +1,6 @@
 import { RpcMessage, RpcMessageRouteType } from '../protocol.js';
 import { cast, ReceiveType, resolveReceiveType } from '@deepkit/type';
+import { base64ToUint8Array } from '@deepkit/core';
 
 export interface RpcHttpRequest {
     headers: { [name: string]: undefined | string | string[] };
@@ -10,7 +11,9 @@ export interface RpcHttpRequest {
 
 export interface RpcHttpResponse {
     setHeader(name: string, value: number | string): this;
+
     writeHead(statusCode: number): this;
+
     end(data?: Uint8Array | string): void;
 }
 
@@ -31,11 +34,11 @@ export class HttpRpcMessage extends RpcMessage {
     }
 
     getSource(): Uint8Array {
-        return Buffer.from(String(this.headers['X-Source']));
+        return base64ToUint8Array(String(this.headers['X-Source']));
     }
 
     getDestination(): Uint8Array {
-        return Buffer.from(String(this.headers['X-Destination']));
+        return base64ToUint8Array(String(this.headers['X-Destination']));
     }
 
     getError(): Error {
@@ -53,7 +56,7 @@ export class HttpRpcMessage extends RpcMessage {
     parseBody<T>(type?: ReceiveType<T>): T {
         const json = this.getJson();
         if (!json) {
-            throw new Error('No body found')
+            throw new Error('No body found');
         }
         return cast(json, undefined, undefined, undefined, resolveReceiveType(type));
     }

packages/rpc/src/server/kernel.ts

@@ -8,7 +8,7 @@
  * You should have received a copy of the MIT License along with this program.
  */
 
-import { arrayRemoveItem, ClassType, getClassName } from '@deepkit/core';
+import { arrayRemoveItem, bufferToString, ClassType, createBuffer, getClassName } from '@deepkit/core';
 import {
     ReceiveType,
     ReflectionKind,
@@ -31,7 +31,6 @@ import {
     RpcTypes,
 } from '../model.js';
 import {
-    createBuffer,
     createRpcCompositeMessage,
     createRpcCompositeMessageSourceDest,
     createRpcMessage,
@@ -466,7 +465,7 @@ export class RpcKernelConnection extends RpcKernelBaseConnection {
                     messageResponse,
                 );
             } else {
-                const body = request.body && request.body.byteLength > 0 ? JSON.parse(Buffer.from(request.body).toString()) : {args: url.searchParams.getAll('arg').map(v => v)};
+                const body = request.body && request.body.byteLength > 0 ? JSON.parse(bufferToString(request.body)) : {args: url.searchParams.getAll('arg').map(v => v)};
                 base.args = body.args || [];
                 await this.actionHandler.handleAction(
                     new HttpRpcMessage(1, false, RpcTypes.Action, RpcMessageRouteType.client, request.headers, base),