=serialization rm 5.2.4 workaround, fix LWWMap with Codable Value (Codable in Proto nesting) (#782)

* =fix,ser serialization would crash if LWWMap used a value that was Codable

* debug CI; this works locally in docker swift 5.2.5 + linux...

* dont install jazzy on xenial

* version

Author: ktoso

Sources/DistributedActors/Serialization/Serialization+Codable.swift

@@ -80,42 +80,3 @@ extension Encodable {
         try encoder.encode(self)
     }
 }
-
-// TODO: could not get it to work
-//// ==== ----------------------------------------------------------------------------------------------------------------
-//// MARK: encode with manifest
-//
-// extension KeyedEncodingContainerProtocol {
-//    mutating func encode<T>(_ value: T, forKey key: Self.Key, forManifestKey manifestKey: Self.Key) throws where T: Encodable {
-//        let encoder = self.superEncoder()
-//        guard let context: Serialization.Context = encoder.actorSerializationContext else {
-//            throw SerializationError.missingSerializationContext(encoder, value)
-//        }
-//
-//        let serialized = try context.serialization.serialize(value)
-//
-//        try self.encode(serialized.manifest, forKey: manifestKey)
-//        try self.encode(serialized.buffer.readData(), forKey: key)
-//    }
-//
-//
-// }
-//
-// extension KeyedDecodingContainerProtocol {
-//    func decode<T>(
-//        _ type: T.Type, forKey key: Self.Key, forManifestKey manifestKey: Self.Key,
-//        file: String = #file, line: UInt = #line
-//    ) throws -> T where T: Decodable {
-//        let decoder = self.superEncoder()
-//
-//        guard let context: Serialization.Context = decoder.actorSerializationContext else {
-//            throw SerializationError.missingSerializationContext(T.self, details: "Missing context", file: file, line: line)
-//        }
-//
-//        let manifest = try self.decode(Serialization.Manifest.self, forKey: manifestKey)
-//        let data = try self.decode(Foundation.Data.self, forKey: manifestKey)
-//
-//        return try context.serialization.deserialize(as: T.self, from: .data(data), using: manifest)
-//    }
-//
-// }

Sources/DistributedActors/Serialization/TopLevelProtobufCoders.swift

@@ -20,8 +20,6 @@ import NIOFoundationCompat
 // ==== ----------------------------------------------------------------------------------------------------------------
 // MARK: Top-Level Bytes-Blob Encoder
 
-// TODO: TopLevelDataEncoder
-
 final class TopLevelProtobufBlobEncoder: _TopLevelBlobEncoder {
     let allocator: ByteBufferAllocator
 
@@ -78,7 +76,7 @@ final class TopLevelProtobufBlobEncoder: _TopLevelBlobEncoder {
 
     func unkeyedContainer() -> UnkeyedEncodingContainer {
         fatalErrorBacktrace("Attempted \(#function) in \(self)")
-        // TopLevelProtobufBlobEncoderContainer(superEncoder: self)
+        // TopLevelProtobufBlobUnkeyedEncodingContainer(superEncoder: self)
     }
 
     func singleValueContainer() -> SingleValueEncodingContainer {
@@ -100,7 +98,9 @@ struct TopLevelProtobufBlobSingleValueEncodingContainer: SingleValueEncodingCont
         case let repr as AnyProtobufRepresentable:
             try repr.encode(to: self.superEncoder)
         case let data as Data:
-            try data.encode(to: self.superEncoder)
+            try self.superEncoder.store(data: data)
+        case let bytes as [UInt8]:
+            try self.superEncoder.store(bytes: bytes)
         default:
             throw SerializationError.unableToSerialize(hint: "Attempted encode \(T.self) into a \(Self.self) which only supports ProtobufRepresentable")
         }

Tests/DistributedActorsTests/CRDT/CRDTGossipReplicationClusteredTests.swift

@@ -32,11 +32,25 @@ final class CRDTGossipReplicationClusteredTests: ClusteredActorSystemsXCTestCase
 
     override func configureActorSystem(settings: inout ActorSystemSettings) {
         settings.serialization.register(CRDT.ORSet<String>.self)
-        settings.serialization.register(CRDT.LWWMap<String, ValueHolder<String?>>.self)
-        settings.serialization.register(CRDT.LWWRegister<ValueHolder<String?>>.self)
-        settings.serialization.register(ValueHolder<String?>.self)
+        settings.serialization.register(CRDT.LWWMap<String, String?>.self)
+        settings.serialization.register(CRDT.LWWRegister<String?>.self)
+        settings.serialization.register(CodableExampleValue.self)
+        settings.serialization.register(CRDT.LWWMap<String, CodableExampleValue>.self)
+        settings.serialization.register(CRDT.LWWRegister<CodableExampleValue>.self)
+        settings.serialization.register(String?.self)
     }
 
+    struct CodableExampleValue: Equatable, Codable, ExpressibleByStringLiteral {
+        let value: String
+
+        init(stringLiteral value: StringLiteralType) {
+            self.value = value
+        }
+    }
+
+    // ==== ----------------------------------------------------------------------------------------------------------------
+    // MARK: Owns Set
+
     enum OwnsSetMessage: NonTransportableActorMessage {
         case insert(String, CRDT.OperationConsistency)
     }
@@ -58,6 +72,9 @@ final class CRDTGossipReplicationClusteredTests: ClusteredActorSystemsXCTestCase
         }
     }
 
+    // ==== ----------------------------------------------------------------------------------------------------------------
+    // MARK: Owns Counter
+
     func ownsCounter(p: ActorTestProbe<CRDT.GCounter>?) -> Behavior<Int> {
         .setup { context in
             let counter: CRDT.ActorOwned<CRDT.GCounter> = CRDT.GCounter.makeOwned(by: context, id: "counter")
@@ -72,6 +89,9 @@ final class CRDTGossipReplicationClusteredTests: ClusteredActorSystemsXCTestCase
         }
     }
 
+    // ==== ----------------------------------------------------------------------------------------------------------------
+    // MARK: Owns Map
+
     enum OwnsMapMessage<Value: Codable>: NonTransportableActorMessage {
         case set(key: String, value: Value, CRDT.OperationConsistency)
     }
@@ -123,15 +143,36 @@ final class CRDTGossipReplicationClusteredTests: ClusteredActorSystemsXCTestCase
         try self.expectSet(probe: p2, expected: ["a", "aa", "b"])
     }
 
-    struct ValueHolder<Value: Codable & Equatable>: Codable, Equatable {
-        let value: Value?
-
-        init(_ value: Value? = nil) {
-            self.value = value
+    func test_gossip_localLWWMapUpdate_toOtherNode() throws {
+        let configure: (inout ActorSystemSettings) -> Void = { settings in
+            settings.crdt.gossipInterval = .seconds(1)
+            settings.crdt.gossipIntervalRandomFactor = 0 // no random factor, exactly 1second intervals
         }
+        let first = self.setUpNode("first", configure)
+        let second = self.setUpNode("second", configure)
+        try self.joinNodes(node: first, with: second, ensureMembers: .up)
+
+        let p1 = self.testKit(first).spawnTestProbe("probe-one", expecting: CRDT.LWWMap<String, String?>.self)
+        let p2 = self.testKit(second).spawnTestProbe("probe-two", expecting: CRDT.LWWMap<String, String?>.self)
+
+        let one = try first.spawn("one", self.ownsLWWMap(p: p1, defaultValue: nil))
+        let two = try second.spawn("two", self.ownsLWWMap(p: p2, defaultValue: nil))
+
+        one.tell(.set(key: "a", value: "foo", .local))
+        one.tell(.set(key: "aa", value: nil, .local))
+
+        let gossipOneExpectMap: [String: String?] = ["a": .init("foo"), "aa": nil]
+        try self.expectMap(probe: p1, expected: gossipOneExpectMap)
+        try self.expectMap(probe: p2, expected: gossipOneExpectMap)
+
+        two.tell(.set(key: "b", value: "bar", .local))
+
+        let gossipTwoExpectMap: [String: String?] = ["a": "foo", "aa": nil, "b": "bar"]
+        try self.expectMap(probe: p1, expected: gossipTwoExpectMap)
+        try self.expectMap(probe: p2, expected: gossipTwoExpectMap)
     }
 
-    func test_gossip_localLWWMapUpdate_toOtherNode() throws {
+    func test_gossip_localLWWMapUpdate_toOtherNode_codableValue() throws {
         let configure: (inout ActorSystemSettings) -> Void = { settings in
             settings.crdt.gossipInterval = .seconds(1)
             settings.crdt.gossipIntervalRandomFactor = 0 // no random factor, exactly 1second intervals
@@ -140,24 +181,22 @@ final class CRDTGossipReplicationClusteredTests: ClusteredActorSystemsXCTestCase
         let second = self.setUpNode("second", configure)
         try self.joinNodes(node: first, with: second, ensureMembers: .up)
 
-        let p1 = self.testKit(first).spawnTestProbe("probe-one", expecting: CRDT.LWWMap<String, ValueHolder<String?>>.self)
-        let p2 = self.testKit(second).spawnTestProbe("probe-two", expecting: CRDT.LWWMap<String, ValueHolder<String?>>.self)
+        let p1 = self.testKit(first).spawnTestProbe("probe-one", expecting: CRDT.LWWMap<String, CodableExampleValue>.self)
+        let p2 = self.testKit(second).spawnTestProbe("probe-two", expecting: CRDT.LWWMap<String, CodableExampleValue>.self)
 
-        // TODO: JSON serialization blows up on Swift 5.2.4 Linux with top-level values so we must wrap (https://bugs.swift.org/browse/SR-13173). Change nilPlaceholder to `:String? = .none` once fixed
-        let nilPlaceholder = ValueHolder<String?>()
-        let one = try first.spawn("one", self.ownsLWWMap(p: p1, defaultValue: nilPlaceholder))
-        let two = try second.spawn("two", self.ownsLWWMap(p: p2, defaultValue: nilPlaceholder))
+        let one = try first.spawn("one", self.ownsLWWMap(p: p1, defaultValue: "<nil>"))
+        let two = try second.spawn("two", self.ownsLWWMap(p: p2, defaultValue: "<nil>"))
 
-        one.tell(.set(key: "a", value: .init("foo"), .local))
-        one.tell(.set(key: "aa", value: nilPlaceholder, .local))
+        one.tell(.set(key: "a", value: "foo", .local))
+        one.tell(.set(key: "aa", value: "baz", .local))
 
-        let gossipOneExpectMap: [String: ValueHolder<String?>] = ["a": .init("foo"), "aa": nilPlaceholder]
+        let gossipOneExpectMap: [String: CodableExampleValue] = ["a": "foo", "aa": "baz"]
         try self.expectMap(probe: p1, expected: gossipOneExpectMap)
         try self.expectMap(probe: p2, expected: gossipOneExpectMap)
 
-        two.tell(.set(key: "b", value: .init("bar"), .local))
+        two.tell(.set(key: "b", value: "bar", .local))
 
-        let gossipTwoExpectMap: [String: ValueHolder<String?>] = ["a": .init("foo"), "aa": nilPlaceholder, "b": .init("bar")]
+        let gossipTwoExpectMap: [String: CodableExampleValue] = ["a": "foo", "aa": "baz", "b": "bar"]
         try self.expectMap(probe: p1, expected: gossipTwoExpectMap)
         try self.expectMap(probe: p2, expected: gossipTwoExpectMap)
     }
@@ -294,11 +333,11 @@ final class CRDTGossipReplicationClusteredTests: ClusteredActorSystemsXCTestCase
         }
     }
 
-    private func expectMap(probe: ActorTestProbe<CRDT.LWWMap<String, ValueHolder<String?>>>, expected: [String: ValueHolder<String?>], file: StaticString = #file, line: UInt = #line) throws {
+    private func expectMap<Value>(probe: ActorTestProbe<CRDT.LWWMap<String, Value>>, expected: [String: Value], file: StaticString = #file, line: UInt = #line) throws {
         let testKit: ActorTestKit = self._testKits.first!
 
         try testKit.eventually(within: .seconds(10)) {
-            let replicated: CRDT.LWWMap<String, ValueHolder<String?>> = try probe.expectMessage(within: .seconds(10), file: file, line: line)
+            let replicated: CRDT.LWWMap<String, Value> = try probe.expectMessage(within: .seconds(10), file: file, line: line)
             pinfo("[\(probe.name)] received updated crdt: \(replicated)")
 
             guard expected == replicated.underlying else {

docker/Dockerfile

@@ -13,12 +13,10 @@ ENV LANG en_US.UTF-8
 ENV LANGUAGE en_US.UTF-8
 
 # dependencies
-RUN apt-get update && apt-get install -y wget
-RUN apt-get update && apt-get install -y lsof dnsutils netcat-openbsd net-tools curl jq # used by integration tests
-
-# ruby and jazzy for docs generation
-RUN apt-get update && apt-get install -y ruby ruby-dev libsqlite3-dev
-RUN gem install jazzy --no-ri --no-rdoc
+RUN apt-get update && apt-get install -y wget \
+    lsof dnsutils netcat-openbsd net-tools curl jq \
+    ruby ruby-dev libsqlite3-dev
+RUN if [ "${ubuntu_version}" != "xenial" ] ; then  gem install jazzy --no-ri --no-rdoc ; fi
 RUN gem install asciidoctor -v 1.5.8 --no-ri --no-rdoc
 RUN gem install asciidoctor-pdf --pre --no-ri --no-rdoc
 RUN gem install asciidoctor-diagram -v 1.5.8 --no-ri --no-rdoc

docker/docker-compose.yaml

@@ -43,7 +43,7 @@ services:
 
   test:
     <<: *common
-    command: /bin/bash -cl "SACT_WARNINGS_AS_ERRORS=yes swift test --enable-test-discovery -Xswiftc -DSACT_TESTS_LEAKS && ./scripts/integration_tests.sh"
+    command: /bin/bash -cl "swift -version; SACT_WARNINGS_AS_ERRORS=yes swift test --enable-test-discovery -Xswiftc -DSACT_TESTS_LEAKS && ./scripts/integration_tests.sh"
 
   bench:
     <<: *common