diff --git a/packages/ssz/package.json b/packages/ssz/package.json index d0df8f1d..21d0a098 100644 --- a/packages/ssz/package.json +++ b/packages/ssz/package.json @@ -21,7 +21,8 @@ "check-types": "tsc --noEmit", "clean": "rm -rf lib && rm -rf dist && rm -f tsconfig.tsbuildinfo", "prepublishOnly": "yarn build", - "benchmark": "node --loader=ts-node/esm --max-old-space-size=4096 --expose-gc ../../node_modules/.bin/benchmark 'test/perf/*.test.ts'", + "benchmark": "yarn benchmark:files 'test/perf/*.test.ts'", + "benchmark:files": "node --loader=ts-node/esm --max-old-space-size=4096 --expose-gc ../../node_modules/.bin/benchmark", "benchmark:local": "yarn benchmark --local", "test:unit": "vitest run --dir test/unit", "test:spec": "yarn test:spec-generic && yarn test:spec-static test:spec-eip-4881", diff --git a/packages/ssz/src/type/abstract.ts b/packages/ssz/src/type/abstract.ts index 17cf6c3a..39e7971b 100644 --- a/packages/ssz/src/type/abstract.ts +++ b/packages/ssz/src/type/abstract.ts @@ -81,7 +81,7 @@ export abstract class Type { /** INTERNAL METHOD: Serialize value to existing output ArrayBuffer views */ abstract value_serializeToBytes(output: ByteViews, offset: number, value: V): number; /** INTERNAL METHOD: Deserialize value from a section of ArrayBuffer views */ - abstract value_deserializeFromBytes(data: ByteViews, start: number, end: number): V; + abstract value_deserializeFromBytes(data: ByteViews, start: number, end: number, reuseBytes?: boolean): V; /** INTERNAL METHOD: Return serialized size of a tree */ abstract tree_serializedSize(node: Node): number; /** INTERNAL METHOD: Serialize tree to existing output ArrayBuffer views */ @@ -121,16 +121,9 @@ export abstract class Type { } /** Deserialize binary data to value */ - deserialize(uint8Array: Uint8Array): V { - // Buffer.prototype.slice does not copy memory, force use Uint8Array.prototype.slice https://github.com/nodejs/node/issues/28087 - // - Uint8Array.prototype.slice: Copy memory, safe to mutate - // - Buffer.prototype.slice: Does NOT copy memory, mutation affects both views - // We could ensure that all Buffer instances are converted to Uint8Array before calling value_deserializeFromBytes - // However doing that in a browser friendly way is not easy. Downstream code uses `Uint8Array.prototype.slice.call` - // to ensure Buffer.prototype.slice is never used. Unit tests also test non-mutability. - + deserialize(uint8Array: Uint8Array, opts?: {reuseBytes?: boolean}): V { const dataView = new DataView(uint8Array.buffer, uint8Array.byteOffset, uint8Array.byteLength); - return this.value_deserializeFromBytes({uint8Array, dataView}, 0, uint8Array.length); + return this.value_deserializeFromBytes({uint8Array, dataView}, 0, uint8Array.length, opts?.reuseBytes); } // Merkleization diff --git a/packages/ssz/src/type/arrayComposite.ts b/packages/ssz/src/type/arrayComposite.ts index 7494e279..6eed76bf 100644 --- a/packages/ssz/src/type/arrayComposite.ts +++ b/packages/ssz/src/type/arrayComposite.ts @@ -90,7 +90,8 @@ export function value_deserializeFromBytesArrayComposite< data: ByteViews, start: number, end: number, - arrayProps: ArrayProps + arrayProps: ArrayProps, + reuseBytes?: boolean ): ValueOf[] { const offsets = readOffsetsArrayComposite(elementType.fixedSize, data.dataView, start, end, arrayProps); const length = offsets.length; // Capture length before pushing end offset @@ -102,7 +103,7 @@ export function value_deserializeFromBytesArrayComposite< // The offsets are relative to the start const startEl = start + offsets[i]; const endEl = i === length - 1 ? end : start + offsets[i + 1]; - values[i] = elementType.value_deserializeFromBytes(data, startEl, endEl); + values[i] = elementType.value_deserializeFromBytes(data, startEl, endEl, reuseBytes); } return values; diff --git a/packages/ssz/src/type/bitList.ts b/packages/ssz/src/type/bitList.ts index 451f3694..46553943 100644 --- a/packages/ssz/src/type/bitList.ts +++ b/packages/ssz/src/type/bitList.ts @@ -6,6 +6,7 @@ import { packedNodeRootsToBytes, packedRootsBytesToNode, } from "@chainsafe/persistent-merkle-tree"; +import {slice} from "../util/byteArray.ts"; import {maxChunksToDepth} from "../util/merkleize.ts"; import {namedClass} from "../util/named.ts"; import {Require} from "../util/types.ts"; @@ -79,8 +80,8 @@ export class BitListType extends BitArrayType { return applyPaddingBit(output.uint8Array, offset, value.bitLen); } - value_deserializeFromBytes(data: ByteViews, start: number, end: number): BitArray { - const {uint8Array, bitLen} = this.deserializeUint8ArrayBitListFromBytes(data.uint8Array, start, end); + value_deserializeFromBytes(data: ByteViews, start: number, end: number, reuseBytes?: boolean): BitArray { + const {uint8Array, bitLen} = this.deserializeUint8ArrayBitListFromBytes(data.uint8Array, start, end, reuseBytes); return new BitArray(uint8Array, bitLen); } @@ -135,8 +136,13 @@ export class BitListType extends BitArrayType { // Deserializer helpers - private deserializeUint8ArrayBitListFromBytes(data: Uint8Array, start: number, end: number): BitArrayDeserialized { - const {uint8Array, bitLen} = deserializeUint8ArrayBitListFromBytes(data, start, end); + private deserializeUint8ArrayBitListFromBytes( + data: Uint8Array, + start: number, + end: number, + reuseBytes?: boolean + ): BitArrayDeserialized { + const {uint8Array, bitLen} = deserializeUint8ArrayBitListFromBytes(data, start, end, reuseBytes); if (bitLen > this.limitBits) { throw Error(`bitLen over limit ${bitLen} > ${this.limitBits}`); } @@ -149,7 +155,8 @@ type BitArrayDeserialized = {uint8Array: Uint8Array; bitLen: number}; export function deserializeUint8ArrayBitListFromBytes( data: Uint8Array, start: number, - end: number + end: number, + reuseBytes?: boolean ): BitArrayDeserialized { if (end > data.length) { throw Error(`BitList attempting to read byte ${end} of data length ${data.length}`); @@ -163,15 +170,13 @@ export function deserializeUint8ArrayBitListFromBytes( } if (lastByte === 1) { - // Buffer.prototype.slice does not copy memory, Enforce Uint8Array usage https://github.com/nodejs/node/issues/28087 - const uint8Array = Uint8Array.prototype.slice.call(data, start, end - 1); + const uint8Array = slice(data, start, end - 1, reuseBytes); const bitLen = (size - 1) * 8; return {uint8Array, bitLen}; } // the last byte is > 1, so a padding bit will exist in the last byte and need to be removed - // Buffer.prototype.slice does not copy memory, Enforce Uint8Array usage https://github.com/nodejs/node/issues/28087 - const uint8Array = Uint8Array.prototype.slice.call(data, start, end); + const uint8Array = slice(data, start, end, reuseBytes); // mask lastChunkByte const lastByteBitLength = lastByte.toString(2).length - 1; const bitLen = (size - 1) * 8 + lastByteBitLength; diff --git a/packages/ssz/src/type/bitVector.ts b/packages/ssz/src/type/bitVector.ts index 3fbd7007..c95f41a6 100644 --- a/packages/ssz/src/type/bitVector.ts +++ b/packages/ssz/src/type/bitVector.ts @@ -1,4 +1,5 @@ import {Node, getNodesAtDepth, packedNodeRootsToBytes, packedRootsBytesToNode} from "@chainsafe/persistent-merkle-tree"; +import {slice} from "../util/byteArray.ts"; import {maxChunksToDepth} from "../util/merkleize.ts"; import {namedClass} from "../util/named.ts"; import {Require} from "../util/types.ts"; @@ -78,10 +79,9 @@ export class BitVectorType extends BitArrayType { return offset + this.fixedSize; } - value_deserializeFromBytes(data: ByteViews, start: number, end: number): BitArray { + value_deserializeFromBytes(data: ByteViews, start: number, end: number, reuseBytes?: boolean): BitArray { this.assertValidLength(data.uint8Array, start, end); - // Buffer.prototype.slice does not copy memory, Enforce Uint8Array usage https://github.com/nodejs/node/issues/28087 - return new BitArray(Uint8Array.prototype.slice.call(data.uint8Array, start, end), this.lengthBits); + return new BitArray(slice(data.uint8Array, start, end, reuseBytes), this.lengthBits); } tree_serializedSize(): number { diff --git a/packages/ssz/src/type/byteArray.ts b/packages/ssz/src/type/byteArray.ts index a382616a..7d041b65 100644 --- a/packages/ssz/src/type/byteArray.ts +++ b/packages/ssz/src/type/byteArray.ts @@ -7,7 +7,7 @@ import { getHashComputations, toGindex, } from "@chainsafe/persistent-merkle-tree"; -import {byteArrayEquals, fromHexString, toHexString} from "../util/byteArray.ts"; +import {byteArrayEquals, fromHexString, slice, toHexString} from "../util/byteArray.ts"; import {ByteViews} from "./abstract.ts"; import {CompositeType, LENGTH_GINDEX} from "./composite.ts"; @@ -74,9 +74,9 @@ export abstract class ByteArrayType extends CompositeType>> extends return variableIndex; } - value_deserializeFromBytes(data: ByteViews, start: number, end: number): ValueOfFields { + value_deserializeFromBytes(data: ByteViews, start: number, end: number, reuseBytes?: boolean): ValueOfFields { const fieldRanges = this.getFieldRanges(data.dataView, start, end); const value = {} as {[K in keyof Fields]: unknown}; for (let i = 0; i < this.fieldsEntries.length; i++) { const {fieldName, fieldType} = this.fieldsEntries[i]; const fieldRange = fieldRanges[i]; - value[fieldName] = fieldType.value_deserializeFromBytes(data, start + fieldRange.start, start + fieldRange.end); + value[fieldName] = fieldType.value_deserializeFromBytes( + data, + start + fieldRange.start, + start + fieldRange.end, + reuseBytes + ); } return value as ValueOfFields; diff --git a/packages/ssz/src/type/listComposite.ts b/packages/ssz/src/type/listComposite.ts index cb461154..d21a0c17 100644 --- a/packages/ssz/src/type/listComposite.ts +++ b/packages/ssz/src/type/listComposite.ts @@ -135,8 +135,13 @@ export class ListCompositeType< return value_serializeToBytesArrayComposite(this.elementType, value.length, output, offset, value); } - value_deserializeFromBytes(data: ByteViews, start: number, end: number): ValueOf[] { - return value_deserializeFromBytesArrayComposite(this.elementType, data, start, end, this); + value_deserializeFromBytes( + data: ByteViews, + start: number, + end: number, + reuseBytes?: boolean + ): ValueOf[] { + return value_deserializeFromBytesArrayComposite(this.elementType, data, start, end, this, reuseBytes); } tree_serializedSize(node: Node): number { diff --git a/packages/ssz/src/type/optional.ts b/packages/ssz/src/type/optional.ts index 81bf1a0b..54bb2b00 100644 --- a/packages/ssz/src/type/optional.ts +++ b/packages/ssz/src/type/optional.ts @@ -124,14 +124,19 @@ export class OptionalType> extends CompositeTy return offset; } - value_deserializeFromBytes(data: ByteViews, start: number, end: number): ValueOfType { + value_deserializeFromBytes( + data: ByteViews, + start: number, + end: number, + reuseBytes?: boolean + ): ValueOfType { if (start === end) return null as ValueOfType; const selector = data.uint8Array[start]; if (selector !== 1) { throw new Error(`Invalid selector for Optional type: ${selector}`); } - return this.elementType.value_deserializeFromBytes(data, start + 1, end) as ValueOfType; + return this.elementType.value_deserializeFromBytes(data, start + 1, end, reuseBytes) as ValueOfType; } tree_serializedSize(node: Node): number { diff --git a/packages/ssz/src/type/profile.ts b/packages/ssz/src/type/profile.ts index 9190a5cf..c889e97c 100644 --- a/packages/ssz/src/type/profile.ts +++ b/packages/ssz/src/type/profile.ts @@ -266,7 +266,7 @@ export class ProfileType>> extends C return variableIndex; } - value_deserializeFromBytes(data: ByteViews, start: number, end: number): ValueOfFields { + value_deserializeFromBytes(data: ByteViews, start: number, end: number, reuseBytes?: boolean): ValueOfFields { const {optionalFields, fieldRanges} = this.getFieldRanges(data, start, end); const value = {} as {[K in keyof Fields]: unknown}; const optionalFieldsLen = optionalFields.uint8Array.length; @@ -280,7 +280,12 @@ export class ProfileType>> extends C continue; } const fieldRange = fieldRanges[i]; - value[fieldName] = fieldType.value_deserializeFromBytes(data, start + fieldRange.start, start + fieldRange.end); + value[fieldName] = fieldType.value_deserializeFromBytes( + data, + start + fieldRange.start, + start + fieldRange.end, + reuseBytes + ); } return value as ValueOfFields; diff --git a/packages/ssz/src/type/stableContainer.ts b/packages/ssz/src/type/stableContainer.ts index cc91005d..5c8b66cb 100644 --- a/packages/ssz/src/type/stableContainer.ts +++ b/packages/ssz/src/type/stableContainer.ts @@ -258,7 +258,7 @@ export class StableContainerType>> e return variableIndex; } - value_deserializeFromBytes(data: ByteViews, start: number, end: number): ValueOfFields { + value_deserializeFromBytes(data: ByteViews, start: number, end: number, reuseBytes?: boolean): ValueOfFields { const {activeFields, fieldRanges} = this.getFieldRanges(data, start, end); const value = {} as {[K in keyof Fields]: unknown}; @@ -270,7 +270,12 @@ export class StableContainerType>> e } const fieldRange = fieldRanges[rangesIx++]; - value[fieldName] = fieldType.value_deserializeFromBytes(data, start + fieldRange.start, start + fieldRange.end); + value[fieldName] = fieldType.value_deserializeFromBytes( + data, + start + fieldRange.start, + start + fieldRange.end, + reuseBytes + ); } return value as ValueOfFields; diff --git a/packages/ssz/src/type/union.ts b/packages/ssz/src/type/union.ts index 9ab2f639..5ccb084a 100644 --- a/packages/ssz/src/type/union.ts +++ b/packages/ssz/src/type/union.ts @@ -141,7 +141,7 @@ export class UnionType[]> extends CompositeType< return this.types[value.selector].value_serializeToBytes(output, offset + 1, value.value); } - value_deserializeFromBytes(data: ByteViews, start: number, end: number): ValueOfTypes { + value_deserializeFromBytes(data: ByteViews, start: number, end: number, reuseBytes?: boolean): ValueOfTypes { const selector = data.uint8Array[start]; if (selector > this.maxSelector) { throw Error(`Invalid selector ${selector}`); @@ -149,7 +149,7 @@ export class UnionType[]> extends CompositeType< return { selector, - value: this.types[selector].value_deserializeFromBytes(data, start + 1, end) as unknown, + value: this.types[selector].value_deserializeFromBytes(data, start + 1, end, reuseBytes) as unknown, } as ValueOfTypes; } diff --git a/packages/ssz/src/type/vectorComposite.ts b/packages/ssz/src/type/vectorComposite.ts index b21f491e..954d0ed2 100644 --- a/packages/ssz/src/type/vectorComposite.ts +++ b/packages/ssz/src/type/vectorComposite.ts @@ -116,8 +116,13 @@ export class VectorCompositeType< return value_serializeToBytesArrayComposite(this.elementType, this.length, output, offset, value); } - value_deserializeFromBytes(data: ByteViews, start: number, end: number): ValueOf[] { - return value_deserializeFromBytesArrayComposite(this.elementType, data, start, end, this); + value_deserializeFromBytes( + data: ByteViews, + start: number, + end: number, + reuseBytes?: boolean + ): ValueOf[] { + return value_deserializeFromBytesArrayComposite(this.elementType, data, start, end, this, reuseBytes); } tree_serializedSize(node: Node): number { diff --git a/packages/ssz/src/util/byteArray.ts b/packages/ssz/src/util/byteArray.ts index 44dc85b0..728e789b 100644 --- a/packages/ssz/src/util/byteArray.ts +++ b/packages/ssz/src/util/byteArray.ts @@ -3,6 +3,19 @@ import {ByteVector} from "../interface.ts"; // Caching this info costs about ~1000 bytes and speeds up toHexString() by x6 const hexByByte = new Array(256); +/** Wrapper to select Uint8Array.slice or Uint8Array.subarray */ +export function slice(data: Uint8Array, start?: number, end?: number, reuseBytes?: boolean): Uint8Array { + // Buffer.prototype.slice does not copy memory, force use Uint8Array.prototype.slice https://github.com/nodejs/node/issues/28087 + // - Uint8Array.prototype.slice: Copy memory, safe to mutate + // - Buffer.prototype.slice: Does NOT copy memory, mutation affects both views + // We could ensure that all Buffer instances are converted to Uint8Array before calling value_deserializeFromBytes + // However doing that in a browser friendly way is not easy. Downstream code uses `Uint8Array.prototype.slice.call` + // to ensure Buffer.prototype.slice is never used. Unit tests also test non-mutability. + return reuseBytes + ? Uint8Array.prototype.subarray.call(data, start, end) + : Uint8Array.prototype.slice.call(data, start, end); +} + export function toHexString(bytes: Uint8Array | ByteVector): string { let hex = "0x"; for (const byte of bytes) { diff --git a/packages/ssz/test/perf/eth2/deserialize.test.ts b/packages/ssz/test/perf/eth2/deserialize.test.ts index 2c06207f..32071ad1 100644 --- a/packages/ssz/test/perf/eth2/deserialize.test.ts +++ b/packages/ssz/test/perf/eth2/deserialize.test.ts @@ -38,6 +38,15 @@ describe("Deserialize frequent eth2 objects", () => { type.deserialize(bytes); }, }); + + bench({ + id: `deserialize ${type.typeName} - struct (reuse bytes)`, + before: () => type.serialize(value), + beforeEach: (bytes) => bytes, + fn: (bytes) => { + type.deserialize(bytes, {reuseBytes: true}); + }, + }); } for (const validatorCount of [300_000]) { diff --git a/packages/ssz/test/unit/uint8Array.test.ts b/packages/ssz/test/unit/uint8Array.test.ts index b35e68b9..ad19d5d4 100644 --- a/packages/ssz/test/unit/uint8Array.test.ts +++ b/packages/ssz/test/unit/uint8Array.test.ts @@ -1,27 +1,26 @@ import {describe, expect, it} from "vitest"; +import {slice} from "../../src/util/byteArray.ts"; -describe("Mutability Buffer, Uint8Array", () => { - it("Ensure Uint8Array.slice copies memory", () => { +describe("slice", () => { + it("Ensure slice copies memory with reuseBytes == false", () => { const len = 64; const index = 54; const newValue = 1; const u1 = new Uint8Array(len); - const u2 = u1.slice(0); + const u2 = slice(u1, 0, u1.length, false); u2[index] = newValue; expect(u1[index]).to.equal(0, "u1 should have original value"); expect(u2[index]).to.equal(newValue, "u2 should have new value"); }); - - // Buffer.prototype.slice does not copy memory, Enforce Uint8Array usage https://github.com/nodejs/node/issues/28087 - it("Ensure Buffer does not copy memory", () => { + it("Ensure slice does not copy memory with reuseBytes == true", () => { const len = 64; const index = 54; const newValue = 1; - const u1 = Buffer.alloc(len, 0); - const u2 = u1.slice(0); + const u1 = new Uint8Array(len); + const u2 = slice(u1, 0, u1.length, true); u2[index] = newValue; expect(u1[index]).to.equal(newValue, "u1 should have new value");