=singleton harden multinode test for singleton

Author: ktoso

MultiNodeTests/DistributedActorsMultiNodeTests/MultiNode+ClusterSingletonTests.swift

@@ -26,7 +26,6 @@ public final class MultiNodeClusterSingletonTests: MultiNodeTestSuite {
     }
 
     public static func configureMultiNodeTest(settings: inout MultiNodeTestSettings) {
-        settings.initialJoinTimeout = .seconds(5)
         settings.dumpNodeLogs = .always
 
         settings.logCapture.excludeGrep = [
@@ -61,9 +60,12 @@ public final class MultiNodeClusterSingletonTests: MultiNodeTestSuite {
         }
 
         try await multiNode.checkPoint("Hosted singleton") // ----------------------------------------------------------
-
         let reply = try await ref.greet(name: "Hello from \(multiNode.system.name)")
         print("[ON: \(multiNode.system.name)] Got reply: \(reply)")
+
+        try await multiNode.checkPoint("Got reply from singleton") // --------------------------------------------------
+        // Since now all nodes have made a message exchange with the singleton, we can exit this process.
+        // This barrier is important in so that we don't exit the host of the singleton WHILE the others are still getting to talking to it.
     }
 
     distributed actor TheSingleton: ClusterSingleton {

MultiNodeTests/DistributedActorsMultiNodeTests/MultiNode+MultiNodeConductorTests.swift

@@ -27,25 +27,12 @@ public final class MultiNodeConductorTests: MultiNodeTestSuite {
     ///
     /// It also eliminates the possibility of "cheating" and a node peeking
     /// at shared state, since the nodes are properly isolated as if in a real cluster.
-    ///
-    /// ## Distributed execution
-    /// To execute the same test across different physical nodes pass a list ofv
-    /// nodes to use when running the test, e.g.v
-    ///
-    /// ```
-    /// swift package multi-node test --deploy 192.168.0.101:22,192.168.0.102:22,192.168.0.103:22
-    /// ```
-    ///
-    /// Which will evenly spread the test nodes across the passed physical worker nodes.
-    /// Actual network will be used, and it remains possible to kill off nodes and logs
-    /// from all nodes are gathered automatically upon test failures.
     public enum Nodes: String, MultiNodeNodes {
         case first
         case second
     }
 
     public static func configureMultiNodeTest(settings: inout MultiNodeTestSettings) {
-        settings.initialJoinTimeout = .seconds(5)
         settings.dumpNodeLogs = .always
 
         settings.logCapture.excludeGrep = [

Sources/DistributedActors/ClusterSystem.swift

@@ -1045,7 +1045,7 @@ extension ClusterSystem {
 
     /// Called during actor deinit/destroy.
     public func resignID(_ id: ActorID) {
-        self.log.warning("Resign actor id", metadata: ["actor/id": "\(id)"])
+        self.log.trace("Resign actor id", metadata: ["actor/id": "\(id)"])
         self.namingLock.withLockVoid {
             self._reservedNames.remove(id)
             if let ref = self._managedRefs.removeValue(forKey: id) {

Sources/DistributedActors/DistributedActor+Internal.swift

@@ -39,11 +39,7 @@ struct AnyDistributedActor: Sendable, Hashable {
 
     @usableFromInline
     func force<T: DistributedActor>(as _: T.Type) -> T {
-//        if let cast = underlying as? T {
-//            return cast
-//        }
-
-        // FIXME: terrible hack, instead just store the id then?
+        // FIXME: hack, instead just store the id then?
         if let resolved = try? T.resolve(id: underlying.id as! T.ID, using: underlying.actorSystem as! T.ActorSystem) {
             return resolved
         }

Sources/MultiNodeTestKit/MultiNodeTestConductor.swift

@@ -217,11 +217,11 @@ extension MultiNodeTestConductor {
 
         let error = MultiNodeCheckPointError(
             nodeName: nodeName,
-            message: "Checkpoint \(checkpoint.name) failed, node [\(nodeName)] became [\(change.status)] and therefore unable to reach the checkpoint!"
+            message: "Checkpoint [\(checkpoint.name)] failed, node [\(nodeName)] became [\(change.status)] and therefore unable to reach the checkpoint!"
         )
 
         for (name, cc) in self.nodesAtCheckPoint {
-            self.log.warning("Checkpoint \(checkpoint.name) failing. Node \(nodeName) became at least [.down]. Failing waiting node [\(name)]", metadata: [
+            self.log.warning("Checkpoint [\(checkpoint.name)] failing. Node \(nodeName) became at least [.down]. Failing waiting node [\(name)]", metadata: [
                 "multiNode/checkpoint/error": "\(error)",
             ])
             cc.resume(throwing: error)

Sources/MultiNodeTestKitRunner/boot+MultiNodeTestKitRunner+Test.swift

@@ -124,8 +124,6 @@ extension MultiNodeTestKitRunnerBoot {
             throw MultiNodeTestNotFound(suite: suite, test: testName)
         }
 
-        print("NODESSSSS: \(multiNodeTest.nodeNames)")
-
         let devNull = try FileHandle(forUpdating: URL(fileURLWithPath: "/dev/null"))
         defer {
             devNull.closeFile()
@@ -178,6 +176,7 @@ extension MultiNodeTestKitRunnerBoot {
                     let testResult = try interpretNodeTestOutput(
                         result,
                         nodeName: nodeName,
+                        multiNodeTest: multiNodeTest,
                         expectedFailureRegex: multiNodeTest.crashRegex,
                         grepper: grepper,
                         settings: settings,

Sources/MultiNodeTestKitRunner/boot+MultiNodeTestKitRunner.swift

@@ -125,13 +125,15 @@ struct MultiNodeTestKitRunnerBoot {
     func interpretNodeTestOutput(
         _ result: Result<ProgramOutput, Error>,
         nodeName: String,
+        multiNodeTest: MultiNodeTest,
         expectedFailureRegex: String?,
         grepper: OutputGrepper,
         settings: MultiNodeTestSettings,
         runResult: RunResult
     ) throws -> InterpretedRunResult {
+        print()
         print(String(repeating: "-", count: 120))
-        print("[multi-node] [\(nodeName)] Captured logs:")
+        print("[multi-node] [\(nodeName)](\(multiNodeTest.fullTestName)) Captured logs:")
         defer {
             print(String(repeating: "=", count: 120))
         }
@@ -143,7 +145,7 @@ struct MultiNodeTestKitRunnerBoot {
             case .failure(let error as MultiNodeProgramError):
                 var reason: String = "MultiNode test failed, output was dumped."
                 for line in error.completeOutput {
-                    log("[\(nodeName)] \(line)")
+                    log("[\(nodeName)](\(multiNodeTest.testName)) \(line)")
 
                     if line.contains("Fatal error: ") {
                         reason = line
@@ -153,7 +155,7 @@ struct MultiNodeTestKitRunnerBoot {
             case .success(let logs):
                 if settings.dumpNodeLogs == .always {
                     for line in logs {
-                        print("[multi-node] [\(nodeName)] \(line)")
+                        print("[multi-node] [\(nodeName)](\(multiNodeTest.testName)) \(line)")
                     }
                 }
                 return .passedAsExpected
@@ -175,13 +177,13 @@ struct MultiNodeTestKitRunnerBoot {
         if outputJoined.range(of: expectedFailureRegex, options: .regularExpression) != nil {
             if settings.dumpNodeLogs == .always {
                 for line in outputLines {
-                    log("[\(nodeName)] \(line)")
+                    log("[\(nodeName)](\(multiNodeTest.testName)) \(line)")
                 }
             }
             return .crashedAsExpected
         } else {
             for line in outputLines {
-                log("[\(nodeName)] \(line)")
+                log("[\(nodeName)](\(multiNodeTest.testName)) \(line)")
             }
             return .crashRegexDidNotMatch(regex: expectedFailureRegex, output: outputJoined)
         }
@@ -203,11 +205,9 @@ struct MultiNodeTestKitRunnerBoot {
     func run() async throws {
         signal(SIGPIPE, SIG_IGN)
 
-        var failedTests = 0
-
+        var summary = MultiNodeTestSummary()
         switch CommandLine.arguments.dropFirst().first {
         case .some("test"):
-            var summary = MultiNodeTestSummary()
             let filter = TestFilter(parse: CommandLine.arguments.dropFirst())
             defer {
                 summary.dump()
@@ -241,7 +241,7 @@ struct MultiNodeTestKitRunnerBoot {
             exit(EXIT_FAILURE)
         }
 
-        exit(CInt(failedTests == 0 ? EXIT_SUCCESS : EXIT_FAILURE))
+        exit(CInt(summary.failedTests == 0 ? EXIT_SUCCESS : EXIT_FAILURE))
     }
 
     func makeAllNodesCommandString(nodeNames: OrderedSet<String>, deployNodes: [(String, Int)]) -> String {