Copy ssz code from nimbus-eth2 and adjust to stand on its own #2
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,250 @@ | ||
| # ssz_serialization | ||
| # Copyright (c) 2018-2021 Status Research & Development GmbH | ||
| # Licensed and distributed under either of | ||
| # * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). | ||
| # * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). | ||
| # at your option. This file may not be copied, modified, or distributed except according to those terms. | ||
|
|
||
| {.push raises: [Defect].} | ||
| {.pragma: raisesssz, raises: [Defect, MalformedSszError, SszSizeMismatchError].} | ||
|
|
||
| ## SSZ serialization for core SSZ types, as specified in: | ||
| # https://github.com/ethereum/consensus-specs/blob/v1.0.1/ssz/simple-serialize.md#serialization | ||
|
|
||
| import | ||
| std/typetraits, | ||
| stew/[endians2, leb128, objects], | ||
| serialization, serialization/testing/tracing, | ||
| ./ssz_serialization/[codec, bitseqs, types] | ||
|
|
||
| export | ||
| serialization, codec, types, bitseqs | ||
|
|
||
| type | ||
| SszReader* = object | ||
| stream: InputStream | ||
|
|
||
| SszWriter* = object | ||
| stream: OutputStream | ||
|
|
||
| SizePrefixed*[T] = distinct T | ||
| SszMaxSizeExceeded* = object of SerializationError | ||
|
|
||
| VarSizedWriterCtx = object | ||
| fixedParts: WriteCursor | ||
| offset: int | ||
|
|
||
| FixedSizedWriterCtx = object | ||
|
|
||
| serializationFormat SSZ | ||
|
|
||
| SSZ.setReader SszReader | ||
| SSZ.setWriter SszWriter, PreferredOutput = seq[byte] | ||
|
|
||
| template sizePrefixed*[TT](x: TT): untyped = | ||
| type T = TT | ||
| SizePrefixed[T](x) | ||
|
|
||
| proc init*(T: type SszReader, | ||
| stream: InputStream): T = | ||
| T(stream: stream) | ||
|
|
||
| proc writeFixedSized(s: var (OutputStream|WriteCursor), x: auto) {.raises: [Defect, IOError].} = | ||
| mixin toSszType | ||
|
|
||
| when x is byte: | ||
| s.write x | ||
| elif x is bool: | ||
| s.write byte(ord(x)) | ||
| elif x is UintN: | ||
| when cpuEndian == bigEndian: | ||
| s.write toBytesLE(x) | ||
| else: | ||
| s.writeMemCopy x | ||
| elif x is array: | ||
| when x[0] is byte: | ||
| trs "APPENDING FIXED SIZE BYTES", x | ||
| s.write x | ||
| else: | ||
| for elem in x: | ||
| trs "WRITING FIXED SIZE ARRAY ELEMENT" | ||
| s.writeFixedSized toSszType(elem) | ||
| elif x is tuple|object: | ||
| enumInstanceSerializedFields(x, fieldName, field): | ||
| trs "WRITING FIXED SIZE FIELD", fieldName | ||
| s.writeFixedSized toSszType(field) | ||
| else: | ||
| unsupported x.type | ||
|
|
||
| template writeOffset(cursor: var WriteCursor, offset: int) = | ||
| write cursor, toBytesLE(uint32 offset) | ||
|
|
||
| template supports*(_: type SSZ, T: type): bool = | ||
| mixin toSszType | ||
| anonConst compiles(fixedPortionSize toSszType(declval T)) | ||
|
|
||
| func init*(T: type SszWriter, stream: OutputStream): T = | ||
| result.stream = stream | ||
|
|
||
| proc writeVarSizeType(w: var SszWriter, value: auto) {.gcsafe, raises: [Defect, IOError].} | ||
|
|
||
| proc beginRecord*(w: var SszWriter, TT: type): auto = | ||
| type T = TT | ||
| when isFixedSize(T): | ||
| FixedSizedWriterCtx() | ||
| else: | ||
| const offset = when T is array|HashArray: len(T) * offsetSize | ||
| else: fixedPortionSize(T) | ||
| VarSizedWriterCtx(offset: offset, | ||
| fixedParts: w.stream.delayFixedSizeWrite(offset)) | ||
|
|
||
| template writeField*(w: var SszWriter, | ||
| ctx: var auto, | ||
| fieldName: string, | ||
| field: auto) = | ||
| mixin toSszType | ||
| when ctx is FixedSizedWriterCtx: | ||
| writeFixedSized(w.stream, toSszType(field)) | ||
| else: | ||
| type FieldType = type toSszType(field) | ||
|
|
||
| when isFixedSize(FieldType): | ||
| writeFixedSized(ctx.fixedParts, toSszType(field)) | ||
| else: | ||
| trs "WRITING OFFSET ", ctx.offset, " FOR ", fieldName | ||
| writeOffset(ctx.fixedParts, ctx.offset) | ||
| let initPos = w.stream.pos | ||
| trs "WRITING VAR SIZE VALUE OF TYPE ", name(FieldType) | ||
| when FieldType is BitList: | ||
| trs "BIT SEQ ", bytes(field) | ||
| writeVarSizeType(w, toSszType(field)) | ||
| ctx.offset += w.stream.pos - initPos | ||
|
|
||
| template endRecord*(w: var SszWriter, ctx: var auto) = | ||
| when ctx is VarSizedWriterCtx: | ||
| finalize ctx.fixedParts | ||
|
|
||
| proc writeSeq[T](w: var SszWriter, value: seq[T]) | ||
| {.raises: [Defect, IOError].} = | ||
| # Please note that `writeSeq` exists in order to reduce the code bloat | ||
| # produced from generic instantiations of the unique `List[N, T]` types. | ||
| when isFixedSize(T): | ||
| trs "WRITING LIST WITH FIXED SIZE ELEMENTS" | ||
| for elem in value: | ||
| w.stream.writeFixedSized toSszType(elem) | ||
| trs "DONE" | ||
| else: | ||
| trs "WRITING LIST WITH VAR SIZE ELEMENTS" | ||
| var offset = value.len * offsetSize | ||
| var cursor = w.stream.delayFixedSizeWrite offset | ||
| for elem in value: | ||
| cursor.writeFixedSized uint32(offset) | ||
| let initPos = w.stream.pos | ||
| w.writeVarSizeType toSszType(elem) | ||
| offset += w.stream.pos - initPos | ||
| finalize cursor | ||
| trs "DONE" | ||
|
|
||
| proc writeVarSizeType(w: var SszWriter, value: auto) {.raises: [Defect, IOError].} = | ||
| trs "STARTING VAR SIZE TYPE" | ||
|
|
||
| when value is HashArray|HashList: | ||
| writeVarSizeType(w, value.data) | ||
| elif value is SingleMemberUnion: | ||
| doAssert value.selector == 0'u8 | ||
| w.writeValue 0'u8 | ||
| w.writeValue value.value | ||
| elif value is List: | ||
| # We reduce code bloat by forwarding all `List` types to a general `seq[T]` proc. | ||
| writeSeq(w, asSeq value) | ||
| elif value is BitList: | ||
| # ATTENTION! We can reuse `writeSeq` only as long as our BitList type is implemented | ||
| # to internally match the binary representation of SSZ BitLists in memory. | ||
| writeSeq(w, bytes value) | ||
| elif value is object|tuple|array: | ||
| trs "WRITING OBJECT OR ARRAY" | ||
| var ctx = beginRecord(w, type value) | ||
| enumerateSubFields(value, field): | ||
| writeField w, ctx, astToStr(field), field | ||
| endRecord w, ctx | ||
| else: | ||
| unsupported type(value) | ||
|
|
||
| proc writeValue*(w: var SszWriter, x: auto) {.gcsafe, raises: [Defect, IOError].} = | ||
| mixin toSszType | ||
| type T = type toSszType(x) | ||
|
|
||
| when isFixedSize(T): | ||
| w.stream.writeFixedSized toSszType(x) | ||
| else: | ||
| w.writeVarSizeType toSszType(x) | ||
|
|
||
| func sszSize*(value: auto): int {.gcsafe, raises: [Defect].} | ||
|
|
||
| func sszSizeForVarSizeList[T](value: openArray[T]): int = | ||
| result = len(value) * offsetSize | ||
| for elem in value: | ||
| result += sszSize(toSszType elem) | ||
|
|
||
| func sszSize*(value: auto): int {.gcsafe, raises: [Defect].} = | ||
| mixin toSszType | ||
| type T = type toSszType(value) | ||
|
|
||
| when isFixedSize(T): | ||
| anonConst fixedPortionSize(T) | ||
|
|
||
| elif T is array|List|HashList|HashArray: | ||
| type E = ElemType(T) | ||
| when isFixedSize(E): | ||
| len(value) * anonConst(fixedPortionSize(E)) | ||
| elif T is HashArray: | ||
| sszSizeForVarSizeList(value.data) | ||
| elif T is array: | ||
| sszSizeForVarSizeList(value) | ||
| else: | ||
| sszSizeForVarSizeList(asSeq value) | ||
|
|
||
| elif T is BitList: | ||
| return len(bytes(value)) | ||
|
|
||
| elif T is SingleMemberUnion: | ||
| sszSize(toSszType value.value) + 1 | ||
|
|
||
| elif T is object|tuple: | ||
| result = anonConst fixedPortionSize(T) | ||
| enumInstanceSerializedFields(value, _{.used.}, field): | ||
| type FieldType = type toSszType(field) | ||
| when not isFixedSize(FieldType): | ||
| result += sszSize(toSszType field) | ||
|
|
||
| else: | ||
| unsupported T | ||
|
|
||
| proc writeValue*[T](w: var SszWriter, x: SizePrefixed[T]) {.raises: [Defect, IOError].} = | ||
| var cursor = w.stream.delayVarSizeWrite(Leb128.maxLen(uint64)) | ||
| let initPos = w.stream.pos | ||
| w.writeValue T(x) | ||
| let length = toBytes(uint64(w.stream.pos - initPos), Leb128) | ||
| cursor.finalWrite length.toOpenArray() | ||
|
|
||
| proc readValue*(r: var SszReader, val: var auto) {. | ||
| raises: [Defect, MalformedSszError, SszSizeMismatchError, IOError].} = | ||
| mixin readSszBytes | ||
| type T = type val | ||
| when isFixedSize(T): | ||
| const minimalSize = fixedPortionSize(T) | ||
| if r.stream.readable(minimalSize): | ||
| readSszBytes(r.stream.read(minimalSize), val) | ||
| else: | ||
| raise newException(MalformedSszError, "SSZ input of insufficient size") | ||
| else: | ||
| # TODO(zah) Read the fixed portion first and precisely measure the | ||
| # size of the dynamic portion to consume the right number of bytes. | ||
| readSszBytes(r.stream.read(r.stream.len.get), val) | ||
|
|
||
| proc readSszBytes*[T](data: openArray[byte], val: var T) {. | ||
| raises: [Defect, MalformedSszError, SszSizeMismatchError].} = | ||
| # Overload `readSszBytes` to perform custom operations on T after | ||
| # deserialization | ||
| mixin readSszValue | ||
| readSszValue(data, val) | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -9,7 +9,12 @@ skipDirs = @["tests"] | |
|
|
||
| requires "nim >= 1.2.0", | ||
| "serialization", | ||
| "json_serialization", | ||
|
There was a problem hiding this comment. this is an .. unfortunate compromise - ie an ssz library shouldn't depend on json, yet nim:s / nim-ser:s import model with open generics and different behavior, even within an application, depending on imports really leaves no other practical option than this - cc @zah There was a problem hiding this comment. for example, importing There was a problem hiding this comment. Err, why is this dependency necessary? The correct action here is just to remove it IMO. The JSON serialization definitions of There was a problem hiding this comment. Ah, you are trying to provide an excuse for why you structured the code this way. I think some problems just have to be addressed at the language level, because these inappropriate compromises can get you only thus far - you cannot fix the I've suggested one possible language solution multiple times, the last one has been here: There was a problem hiding this comment. well, in fantasy-land, anything is possible. In Nim, the best thing that could happen with the current, arguably broken nim-ser design, would actually be that json-ser includes serializers for all std types it supports out-of-the box without magic imports - or at least the "common" ones, and libraries that are used with nim-json-ser or some other ser always declare support for those serialization formats in the same module as the type itself is declared - anything else doesn't work, in actual real code - for this reason, the best thing this library can do is to depend on json-ser because json-ser is more "commonly" used than ssz. The other, better option is that nim-ser and json-ser no longer contain generic catch-alls for objects at all: every type you want to serialize, you are forced to declare support for it up-front - this doesn't require any language changes: it simply requires that under no circumstances is there something that matches This way, when you "forget" to import the right serializer, you get a compile error instead of a runtime explosion. It's 100% wrong that objects are automatically serialized this way - the only thing that does is make the simple trivial action of adding an import easier, while making the already difficult task of remembering what needs to be imported where impossible. From a design perspective of any library, that's simply wrong - hard things should never be made be made harder, and it's really no loss that if you want to add json-ser support for your type, you need to tell the library "hey, please support this type for me". It can be as trivial as a |
||
| "stew", | ||
| "stint", | ||
| "nimcrypto", | ||
| "blscurve", | ||
| "unittest2" | ||
|
|
||
| proc test(env, path: string) = | ||
| # Compilation language is controlled by TEST_LANG | ||
|
|
@@ -20,8 +25,8 @@ proc test(env, path: string) = | |
| if not dirExists "build": | ||
| mkDir "build" | ||
| exec "nim " & lang & " " & env & | ||
| " -r --hints:off --warnings:on " & path | ||
|
|
||
| task test, "Run all tests": | ||
| test "--threads:off -d:PREFER_BLST_SHA256=false", "tests/test_all" | ||
|
There was a problem hiding this comment. Building it currently only using the sha256 implementation of nimcrypto as I'm having issues with BLST outside of the nimbus-eth2 env. |
||
| test "--threads:on -d:PREFER_BLST_SHA256=false", "tests/test_all" | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
this was really intended as an eth2 specific thing - but one thought I've had is to remove nim-serialization completely from the process so as to have a stand-alone SSZ reading library that operates on
openArray- thennim-sercan be glued on top for any .. uh .. advanced use cases should they materialize.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The
codecmodule already provides this (reading that only depends on the compile-time type metadata aspects of nim-serialization such as thedontSerializepragma). I don't see any good reason for aiming to drop this dependency.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
indeed,
codecwas written with this in mind andreadSszBytesas well, to a certain extent, and it should perhaps be moved there so that when using onlyreadSszBytes, the symbol table isn't infected with the rest ofnim-ser- the aim is to get rid of some of the generic instantiation code and other reader/writer magic that isn't really relevant when reading fromopenArray- this is also "usually" a good way to factor this kind of code into orthogonal responsibilities anyway: one layer that deals with concrete raw byte stuff and another layer that deals with magic object transformations and high-level representations and opinions about what should be magic and implicit and what shouldn't be.