[WIP] New multi-node infrastructure for integration tests

InternalPlugins/FishyDocs/Plugins/FishyDocsPlugin/plugin.swift

@@ -1,3 +1,17 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift Distributed Actors open source project
+//
+// Copyright (c) 2022 Apple Inc. and the Swift Distributed Actors project authors
+// Licensed under Apache License v2.0
+//
+// See LICENSE.txt for license information
+// See CONTRIBUTORS.md for the list of Swift Distributed Actors project authors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+//===----------------------------------------------------------------------===//
+
 import Foundation
 import PackagePlugin
 

InternalPlugins/FishyDocs/Sources/FishyDocs/FishyDocs.swift → InternalPlugins/FishyDocs/Sources/FishyDocs/FishyDocsMain.swift

@@ -14,7 +14,7 @@ struct Commands {
 }
 
 @main
-struct FishyDocs: ParsableCommand {
+struct FishyDocsMain: ParsableCommand {
     @Option(help: "Folder containing the docc documentation to scan for fishy-docs")
     var doccFile: String
 

InternalPlugins/MultiNodeTest/.gitignore

@@ -0,0 +1,9 @@
+.DS_Store
+/.build
+/Packages
+/*.xcodeproj
+xcuserdata/
+DerivedData/
+.swiftpm/config/registries.json
+.swiftpm/xcode/package.xcworkspace/contents.xcworkspacedata
+.netrc

InternalPlugins/MultiNodeTest/Package.swift

@@ -0,0 +1,36 @@
+// swift-tools-version: 5.7
+// The swift-tools-version declares the minimum version of Swift required to build this package.
+
+import PackageDescription
+
+let package = Package(
+    name: "MultiNodeTestPlugin",
+    platforms: [
+        .iOS(.v16),
+        .macOS(.v13),
+        .tvOS(.v16),
+        .watchOS(.v9),
+    ],
+    products: [
+        .plugin(
+            name: "MultiNodeTestPlugin",
+            targets: [
+                "MultiNodeTestPlugin",
+            ]
+        ),
+    ],
+    dependencies: [
+        .package(name: "swift-distributed-actors", path: "../"),
+    ],
+    targets: [
+        .plugin(
+            name: "MultiNodeTestPlugin",
+            capability: .command(
+                intent: .custom(verb: "multi-node", description: "Run MultiNodeTestKit based tests across multiple processes or physical compute nodes")
+                /* permissions: needs full network access */
+            ),
+            dependencies: [
+            ]
+        ),
+    ]
+)

InternalPlugins/MultiNodeTest/Plugins/MultiNodeTestPlugin/plugin.swift

@@ -0,0 +1,161 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift Distributed Actors open source project
+//
+// Copyright (c) 2022 Apple Inc. and the Swift Distributed Actors project authors
+// Licensed under Apache License v2.0
+//
+// See LICENSE.txt for license information
+// See CONTRIBUTORS.md for the list of Swift Distributed Actors project authors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+//===----------------------------------------------------------------------===//
+
+import Foundation
+import PackagePlugin
+
+func log(_ log: String, file: String = #fileID, line: UInt = #line) {
+    print("[multi-node] \(log)")
+}
+
+@main
+final class MultiNodeTestPlugin: CommandPlugin {
+    var buildConfiguration: PackageManager.BuildConfiguration = .release
+
+    func performCommand(context: PluginContext, arguments: [String]) async throws {
+        guard arguments.count > 0 else {
+            try self.usage(arguments: arguments)
+        }
+
+        if let configIdx = arguments.firstIndex(of: "-c") {
+            let configValueIdx = arguments.index(after: configIdx)
+            if configValueIdx < arguments.endIndex {
+                if arguments[configValueIdx] == "debug" {
+                    self.buildConfiguration = .debug
+                } else if arguments[configValueIdx] == "release" {
+                    self.buildConfiguration = .release
+                }
+            }
+        }
+
+        // Kill all previous runners
+        Process.killall(name: "MultiNodeTestKitRunner")
+
+        switch self.buildConfiguration {
+        case .debug:
+            log("Building multi-node project for debugging...")
+        case .release:
+            log("Building multi-node project for production...")
+        default:
+            fatalError("Unknown build configuration: \(self.buildConfiguration)")
+        }
+
+        let buildResult = try packageManager.build(
+            .all(includingTests: true),
+            parameters: .init(configuration: self.buildConfiguration)
+        )
+
+        guard buildResult.succeeded else {
+            log(buildResult.logText)
+            return
+        }
+
+        let multiNodeRunner = buildResult.builtArtifacts
+            .filter { $0.kind == .executable }
+            .first { $0.path.lastComponent.starts(with: "MultiNodeTestKitRunner") }
+        guard let multiNodeRunner = multiNodeRunner else {
+            throw MultiNodeTestPluginError(message: "Failed")
+        }
+
+        log("Detected multi-node test runner: \(multiNodeRunner.path.lastComponent)")
+
+        let process = Process()
+        process.binaryPath = "/usr/bin/swift"
+        process.arguments = ["run", "MultiNodeTestKitRunner"]
+        for arg in arguments {
+            process.arguments?.append(arg)
+        }
+
+        do {
+            log("> swift \(process.arguments?.joined(separator: " ") ?? "")")
+            try process.runProcess()
+            process.waitUntilExit()
+        } catch {
+            log("[error] Failed to execute multi-node [\(process.binaryPath!) \(process.arguments!)]! Error: \(error)")
+        }
+    }
+
+    func usage(arguments: [String]) throws -> Never {
+        throw UsageError(message: """
+        ILLEGAL INVOCATION: \(arguments)
+        USAGE:
+        > swift package --disable-sandbox multi-node [OPTIONS] COMMAND
+
+        OPTIONS:
+            -c release/debug  - to build in release or debug mode (default: \(self.buildConfiguration))
+
+        COMMAND:
+            test - run multi-node tests
+            _exec
+        """)
+    }
+}
+
+struct UsageError: Error, CustomStringConvertible {
+    let message: String
+
+    var description: String {
+        self.message
+    }
+}
+
+// Compatible with Swift on all macOS versions as well as Linux
+extension Process {
+    var binaryPath: String? {
+        get {
+            if #available(macOS 10.13, /* Linux */ *) {
+                return self.executableURL?.path
+            } else {
+                return self.launchPath
+            }
+        }
+        set {
+            if #available(macOS 10.13, /* Linux */ *) {
+                self.executableURL = newValue.map { URL(fileURLWithPath: $0) }
+            } else {
+                self.launchPath = newValue
+            }
+        }
+    }
+
+    func runProcess() throws {
+        if #available(macOS 10.13, *) {
+            try self.run()
+        } else {
+            self.launch()
+        }
+    }
+}
+
+extension Process {
+    static func killall(name: String) {
+        let killAllRunners = Process()
+        killAllRunners.binaryPath = "/usr/bin/killall"
+        killAllRunners.arguments = ["-9", "MultiNodeTestKitRunner"]
+        try? killAllRunners.runProcess()
+        killAllRunners.waitUntilExit()
+    }
+}
+
+struct MultiNodeTestPluginError: Error {
+    let message: String
+    let file: String
+    let line: UInt
+
+    init(message: String, file: String = #file, line: UInt = #line) {
+        self.message = message
+        self.file = file
+        self.line = line
+    }
+}

InternalPlugins/MultiNodeTest/README.md

@@ -0,0 +1,3 @@
+# MultiNodeTest Plugin
+
+Allows running tests implemented using `MultiNodeTestKit` across different processes and even physical nodes.

MultiNodeTests/DistributedActorsMultiNodeTests/MultiNode+ClusterSingletonTests.swift

@@ -0,0 +1,85 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift Distributed Actors open source project
+//
+// Copyright (c) 2022 Apple Inc. and the Swift Distributed Actors project authors
+// Licensed under Apache License v2.0
+//
+// See LICENSE.txt for license information
+// See CONTRIBUTORS.md for the list of Swift Distributed Actors project authors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+//===----------------------------------------------------------------------===//
+
+import DistributedActors
+import MultiNodeTestKit
+
+public final class MultiNodeClusterSingletonTests: MultiNodeTestSuite {
+    public init() {}
+
+    public enum Nodes: String, MultiNodeNodes {
+        case first
+        case second
+        case third
+        case fourth
+    }
+
+    public static func configureMultiNodeTest(settings: inout MultiNodeTestSettings) {
+        settings.dumpNodeLogs = .always
+
+        settings.logCapture.excludeGrep = [
+            "SWIMActor.swift", "SWIMInstance.swift",
+            "OperationLogDistributedReceptionist.swift",
+            "Gossiper+Shell.swift",
+        ]
+
+        settings.installPrettyLogger = true
+    }
+
+    public static func configureActorSystem(settings: inout ClusterSystemSettings) {
+        settings.logging.logLevel = .debug
+
+        settings.autoLeaderElection = .lowestReachable(minNumberOfMembers: 3)
+        // settings += ClusterSingletonPlugin() // already installed by MultiNodeTestSuite
+    }
+
+    static var singletonSettings: ClusterSingletonSettings {
+        var settings = ClusterSingletonSettings()
+        settings.allocationStrategy = .byLeadership
+        settings.allocationTimeout = .seconds(15)
+        return settings
+    }
+
+    public let test_singleton_fromManyNodes = MultiNodeTest(MultiNodeClusterSingletonTests.self) { multiNode in
+        let singletonName = "the-one"
+
+        // All nodes run the same code to "host" the singleton (with a different greeting each)
+        let ref = try await multiNode.system.singleton.host(name: singletonName, settings: MultiNodeClusterSingletonTests.singletonSettings) { actorSystem in
+            TheSingleton(greeting: "Hello-\(actorSystem.name)", actorSystem: actorSystem)
+        }
+
+        try await multiNode.checkPoint("Hosted singleton") // ----------------------------------------------------------
+        let reply = try await ref.greet(name: "Hello from \(multiNode.system.name)")
+        print("[ON: \(multiNode.system.name)] Got reply: \(reply)")
+
+        try await multiNode.checkPoint("Got reply from singleton") // --------------------------------------------------
+        // Since now all nodes have made a message exchange with the singleton, we can exit this process.
+        // This barrier is important in so that we don't exit the host of the singleton WHILE the others are still getting to talking to it.
+    }
+
+    distributed actor TheSingleton: ClusterSingleton {
+        typealias ActorSystem = ClusterSystem
+
+        private let greeting: String
+
+        init(greeting: String, actorSystem: ActorSystem) {
+            self.actorSystem = actorSystem
+            self.greeting = greeting
+        }
+
+        distributed func greet(name: String) -> String {
+            "\(self.greeting) \(name)! (from node: \(self.id.uniqueNode), id: \(self.id.detailedDescription))"
+        }
+    }
+}

MultiNodeTests/DistributedActorsMultiNodeTests/MultiNode+MultiNodeConductorTests.swift

@@ -0,0 +1,72 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift Distributed Actors open source project
+//
+// Copyright (c) 2022 Apple Inc. and the Swift Distributed Actors project authors
+// Licensed under Apache License v2.0
+//
+// See LICENSE.txt for license information
+// See CONTRIBUTORS.md for the list of Swift Distributed Actors project authors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+//===----------------------------------------------------------------------===//
+
+import DistributedActors
+import MultiNodeTestKit
+
+/// Tests of the ``MultiNodeTestConductor`` itself.
+public final class MultiNodeConductorTests: MultiNodeTestSuite {
+    public init() {}
+
+    /// Spawns two nodes: first and second, and forms a cluster with them.
+    ///
+    /// ## Default execution
+    /// Unlike normal unit tests, each node is spawned in a separate process,
+    /// allowing is to kill nodes harshly by killing entire processes.
+    ///
+    /// It also eliminates the possibility of "cheating" and a node peeking
+    /// at shared state, since the nodes are properly isolated as if in a real cluster.
+    public enum Nodes: String, MultiNodeNodes {
+        case first
+        case second
+    }
+
+    public static func configureMultiNodeTest(settings: inout MultiNodeTestSettings) {
+        settings.dumpNodeLogs = .always
+
+        settings.logCapture.excludeGrep = [
+            "SWIMActor.swift", "SWIMInstance.swift",
+            "OperationLogDistributedReceptionist.swift",
+            "Gossiper+Shell.swift",
+        ]
+
+        settings.installPrettyLogger = true
+    }
+
+    public static func configureActorSystem(settings: inout ClusterSystemSettings) {
+//        settings.logging.logLevel = .debug
+    }
+
+    public let testCrashSecondNode = MultiNodeTest(MultiNodeConductorTests.self) { multiNode in
+        // A checkPoint suspends until all nodes have reached it, and then all nodes resume execution.
+        try await multiNode.checkPoint("initial")
+
+        // We can execute code only on a specific node:
+        multiNode.log.info("Before RUN ON SECOND ------")
+        try await multiNode.runOn(.second) { second in
+            multiNode.log.info("Before RUN INSIDE SECOND ------")
+            multiNode.log.info("SECOND SHUTDOWN")
+            try second.shutdown()
+            try await second.terminated
+            return
+        }
+
+        try await multiNode.runOn(.first) { first in
+            try await first.cluster.waitFor(multiNode[.second], .down, within: .seconds(30))
+        }
+
+        try multiNode.system.shutdown()
+        try await multiNode.system.terminated
+    }
+}