Make remote call errors configurable (#1020)

* Make remote call errors configurable

Resolves #932

* Hide enum, use OID

* Add test for allowed but unregistered error type

* Update docs

* fix docc syntax

Author: yim-lee

Sources/DistributedActors/ClusterSystem.swift

@@ -1207,7 +1207,7 @@ extension ClusterSystem {
     {
         let callID = UUID()
 
-        let timeout = RemoteCall.timeout ?? self.settings.defaultRemoteCallTimeout
+        let timeout = RemoteCall.timeout ?? self.settings.remoteCall.defaultTimeout
         let timeoutTask: Task<Void, Error> = Task.detached {
             try await Task.sleep(nanoseconds: UInt64(timeout.nanoseconds))
             guard !Task.isCancelled else {
@@ -1491,11 +1491,21 @@ public struct ClusterInvocationResultHandler: DistributedTargetInvocationResultH
 
         case .remoteCall(let system, let callID, let channel, let recipient):
             system.log.debug("Result handler, onThrow: \(error)", metadata: ["call/id": "\(callID)"])
+
+            let errorType = type(of: error as Any)
             let reply: RemoteCallReply<_Done>
+
             if let codableError = error as? (Error & Codable) {
-                reply = .init(callID: callID, error: codableError)
+                switch system.settings.remoteCall.codableErrorAllowance.underlying {
+                case .custom(let allowedTypeOIDs) where allowedTypeOIDs.contains(ObjectIdentifier(errorType)):
+                    reply = .init(callID: callID, error: codableError)
+                case .all: // compiler gets confused if this is grouped together with above
+                    reply = .init(callID: callID, error: codableError)
+                default:
+                    reply = .init(callID: callID, error: GenericRemoteCallError(errorType: errorType))
+                }
             } else {
-                reply = .init(callID: callID, error: GenericRemoteCallError(message: "Remote call error of [\(type(of: error as Any))] type occurred"))
+                reply = .init(callID: callID, error: GenericRemoteCallError(errorType: errorType))
             }
             try await channel.writeAndFlush(TransportEnvelope(envelope: Payload(payload: .message(reply)), recipient: recipient))
         }
@@ -1585,6 +1595,14 @@ struct RemoteCallReply<Value: Codable>: AnyRemoteCallReply {
 
 public struct GenericRemoteCallError: Error, Codable {
     public let message: String
+
+    init(message: String) {
+        self.message = message
+    }
+
+    init(errorType: Any.Type) {
+        self.message = "Remote call error of [\(errorType)] type occurred"
+    }
 }
 
 public enum ClusterSystemError: DistributedActorSystemError {

Sources/DistributedActors/ClusterSystemSettings.swift

@@ -167,11 +167,8 @@ public struct ClusterSystemSettings {
     // ==== ------------------------------------------------------------------------------------------------------------
     // MARK: Distributed Actor Calls
 
-    /// If no other timeout is specified, this timeout is applied to every distributed call.
-    /// A "distributed call" is any function call of a distributed function on a 'remote' distributed actor.
-    ///
-    /// Set to `.effectivelyInfinite` to avoid setting a timeout, although this is not recommended.
-    public var defaultRemoteCallTimeout: Duration = .seconds(5)
+    /// A "remote call" is any function call of a `distributed` function on a 'remote' distributed actor.
+    public var remoteCall: RemoteCallSettings = .default
 
     // ==== ------------------------------------------------------------------------------------------------------------
     // MARK: TLS & Security settings
@@ -449,3 +446,48 @@ public struct ServiceDiscoverySettings {
         case dynamic(AnyServiceDiscovery)
     }
 }
+
+// ==== ----------------------------------------------------------------------------------------------------------------
+// MARK: Remote Call Settings
+
+extension ClusterSystemSettings {
+    public struct RemoteCallSettings {
+        public static var `default`: RemoteCallSettings {
+            .init()
+        }
+
+        /// If no other timeout is specified, this timeout is applied to every distributed call.
+        ///
+        /// Set to `.effectivelyInfinite` to avoid setting a timeout, although this is not recommended.
+        public var defaultTimeout: Duration = .seconds(5)
+
+        public var codableErrorAllowance: CodableErrorAllowanceSettings = .all
+
+        public struct CodableErrorAllowanceSettings {
+            internal enum CodableErrorAllowance {
+                case none
+                case all
+                // OIDs of allowed types
+                case custom(Set<ObjectIdentifier>)
+            }
+
+            internal let underlying: CodableErrorAllowance
+
+            internal init(allowance: CodableErrorAllowance) {
+                self.underlying = allowance
+            }
+
+            /// All ``Codable`` errors will be converted to ``GenericRemoteCallError``.
+            public static let none: CodableErrorAllowanceSettings = .init(allowance: .none)
+
+            /// All ``Codable`` errors will be returned as-is.
+            public static let all: CodableErrorAllowanceSettings = .init(allowance: .all)
+
+            /// Only the indicated ``Codable`` errors are allowed. Others are converted to ``GenericRemoteCallError``.
+            public static func custom(allowedTypes: [(Error & Codable).Type]) -> CodableErrorAllowanceSettings {
+                let oids = allowedTypes.map { ObjectIdentifier($0) }
+                return .init(allowance: .custom(Set(oids)))
+            }
+        }
+    }
+}

Sources/DistributedActors/DistributedActors.docc/Clustering.md

@@ -98,7 +98,7 @@ Similarly, you can implement the [ServiceDiscovery](https://github.com/apple/swi
 and this will then enable the cluster to locate nodes to contact and join automatically. It also benefits all other uses of service discovery in such new environment,
 so we encourage publishing your implementations if you're able to!
 
-## Cluster events
+## Cluster Events
 
 Cluster events are events emitted by the cluster as changes happen to the lifecycle of members of the cluster.
 
@@ -273,19 +273,55 @@ distributed actor Boss: LifecycleWatch {
 
 Remote calls are at the heart of what makes distributed actors actually distributed.
 
-A call made on a remote distributed actor reference, will cross network boundaries, and therefore may way due to 
+A call made on a remote distributed actor reference will cross network boundaries, and therefore may fail due to 
 network issues, message loss, serialization errors, or other reasons such as the recipient node crashing as it 
 processes the message. Even replies to remote calls could sometimes fail being delivered, so you might need to 
 design your distributed actors with idempotency (the resilience of a method being called more than once, e.g. due to a retry) in mind.
 
 By default, to avoid "hanging" a remote caller forever on a suspended remote call as the recipient node fails to reply to it,
-for example because it (or the network itself), are currently unresponsive, remote calls have a default timeout configured,
-and if no reply is received within this duration, the call will fail with a ``RemoteCallError``.
+for example because it (or the network itself), is currently unresponsive, remote calls have a default timeout configured.
+If no reply is received within this duration, the call will fail with a ``RemoteCallError/timedOut``.
 
 You can configure the default timeout used by the cluster system during its initialization:
 
 ```swift
 ClusterSystem() { settings in 
-    settings.
+    settings.remoteCall.defaultTimeout = .seconds(3)
+}
+```
+
+You can override the default timeout for a specific remote call:
+
+```swift
+try await RemoteCall.with(timeout: .seconds(5)) {
+    try await worker.work()
+}
+```
+
+### Remote call errors
+
+By default, if a remote call results in an error that is ``Codable``, the error is returned as-is. Non-``Codable`` errors are 
+converted to ``GenericRemoteCallError``.
+
+You may restrict which ``Codable`` errors get sent back to the caller through configuration:
+
+```swift
+ClusterSystem() { settings in 
+    // By default, all ``Codable`` errors are allowed.
+    settings.remoteCall.codableErrorAllowance = .all
+}
+```
+
+```swift
+ClusterSystem() { settings in 
+    // Only specific types are allowed. All others are returned as ``GenericRemoteCallError``.
+    settings.remoteCall.codableErrorAllowance = .custom(allowedTypes: [SomeCodableError.self, AnotherCodableError.self, ...])
+}
+```
+
+```swift
+ClusterSystem() { settings in 
+    // All errors are returned as ``GenericRemoteCallError``.
+    settings.remoteCall.codableErrorAllowance = .none
 }
 ```

Sources/DistributedActors/Plugins/ClusterSingleton/ClusterSingletonSettings.swift

@@ -41,11 +41,10 @@ public struct ClusterSingletonSettings {
 /// Singleton node allocation strategies.
 public struct AllocationStrategySettings {
     private enum AllocationStrategy {
-        /// Singletons will run on the cluster leader. *All* nodes are potential candidates.
         case byLeadership
     }
 
-    private var allocationStrategy: AllocationStrategy
+    private let allocationStrategy: AllocationStrategy
 
     private init(allocationStrategy: AllocationStrategy) {
         self.allocationStrategy = allocationStrategy
@@ -58,5 +57,6 @@ public struct AllocationStrategySettings {
         }
     }
 
+    /// Singletons will run on the cluster leader. *All* nodes are potential candidates.
     public static let byLeadership: AllocationStrategySettings = .init(allocationStrategy: .byLeadership)
 }