From 18163505e07c40c1d6961bc7e609cc24b6e6702e Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Tue, 25 Feb 2025 17:08:34 -0500 Subject: [PATCH 1/4] Rename `ExitTestArtifacts` and split `ExitCondition` in twain. This PR renames `ExitTestArtifacts` to `ExitTest.Result` and splits `ExitCondition` into two types: `ExitTest.Condition` which can be passed to `#expect(exitsWith:)` and `StatusAtExit` which represents the raw, possibly platform-specific status reported by the kernel when a child process terminates. The latter type is not nested in `ExitTest` because it can be used independently of exit tests and we may want to use it in the future for things like multi-process parallelization, but if a platform supports spawning processes but not exit tests, nesting it in `ExitTest` would make it unavailable. I considered several names for `StatusAtExit`: - `ExitStatus`: too easily confusable with exit _codes_ such as `EXIT_SUCCESS`; - `ProcessStatus`: we don't say "process" in our API surface elsewhere; - `Status`: too generic - `ExitReason`: "status" is a more widely-used term of art for this concept. Foundation uses `terminationStatus` to represent the raw integer value and `Process.TerminationReason` to represent whether it's an exit code or signal. We don't use "termination" in Swift Testing's API anywhere. I settled on `StatusAtExit` because it was distinct and makes it clear that it represents the status of a process _at exit time_ (as opposed to while running, e.g. `enum ProcessStatus { case running; case suspended; case terminated }`. Updates to the exit tests proposal document will follow in a separate PR. --- Sources/Testing/CMakeLists.txt | 5 +- Sources/Testing/ExitTests/ExitCondition.swift | 236 ------------------ .../ExitTests/ExitTest.Condition.swift | 148 +++++++++++ .../Testing/ExitTests/ExitTest.Result.swift | 86 +++++++ Sources/Testing/ExitTests/ExitTest.swift | 57 +++-- .../Testing/ExitTests/ExitTestArtifacts.swift | 90 ------- Sources/Testing/ExitTests/SpawnProcess.swift | 2 +- Sources/Testing/ExitTests/StatusAtExit.swift | 73 ++++++ Sources/Testing/ExitTests/WaitFor.swift | 14 +- .../Expectations/Expectation+Macro.swift | 28 +-- .../ExpectationChecking+Macro.swift | 8 +- Sources/TestingMacros/ConditionMacro.swift | 8 +- Tests/TestingTests/ExitTestTests.swift | 72 ++---- 13 files changed, 385 insertions(+), 442 deletions(-) delete mode 100644 Sources/Testing/ExitTests/ExitCondition.swift create mode 100644 Sources/Testing/ExitTests/ExitTest.Condition.swift create mode 100644 Sources/Testing/ExitTests/ExitTest.Result.swift delete mode 100644 Sources/Testing/ExitTests/ExitTestArtifacts.swift create mode 100644 Sources/Testing/ExitTests/StatusAtExit.swift diff --git a/Sources/Testing/CMakeLists.txt b/Sources/Testing/CMakeLists.txt index e40cb1b0b..2f1e94c1a 100644 --- a/Sources/Testing/CMakeLists.txt +++ b/Sources/Testing/CMakeLists.txt @@ -31,10 +31,11 @@ add_library(Testing Events/Recorder/Event.JUnitXMLRecorder.swift Events/Recorder/Event.Symbol.swift Events/TimeValue.swift - ExitTests/ExitCondition.swift ExitTests/ExitTest.swift - ExitTests/ExitTestArtifacts.swift + ExitTests/ExitTest.Condition.swift + ExitTests/ExitTest.Result.swift ExitTests/SpawnProcess.swift + ExitTests/StatusAtExit.swift ExitTests/WaitFor.swift Expectations/Expectation.swift Expectations/Expectation+Macro.swift diff --git a/Sources/Testing/ExitTests/ExitCondition.swift b/Sources/Testing/ExitTests/ExitCondition.swift deleted file mode 100644 index 19f884303..000000000 --- a/Sources/Testing/ExitTests/ExitCondition.swift +++ /dev/null @@ -1,236 +0,0 @@ -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2024 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for Swift project authors -// - -private import _TestingInternals - -/// An enumeration describing possible conditions under which a process will -/// exit. -/// -/// Values of this type are used to describe the conditions under which an exit -/// test is expected to pass or fail by passing them to -/// ``expect(exitsWith:observing:_:sourceLocation:performing:)`` or -/// ``require(exitsWith:observing:_:sourceLocation:performing:)``. -@_spi(Experimental) -#if SWT_NO_PROCESS_SPAWNING -@available(*, unavailable, message: "Exit tests are not available on this platform.") -#endif -public enum ExitCondition: Sendable { - /// The process terminated successfully with status `EXIT_SUCCESS`. - public static var success: Self { - // Strictly speaking, the C standard treats 0 as a successful exit code and - // potentially distinct from EXIT_SUCCESS. To my knowledge, no modern - // operating system defines EXIT_SUCCESS to any value other than 0, so the - // distinction is academic. - .exitCode(EXIT_SUCCESS) - } - - /// The process terminated abnormally with any status other than - /// `EXIT_SUCCESS` or with any signal. - case failure - - /// The process terminated with the given exit code. - /// - /// - Parameters: - /// - exitCode: The exit code yielded by the process. - /// - /// The C programming language defines two [standard exit codes](https://en.cppreference.com/w/c/program/EXIT_status), - /// `EXIT_SUCCESS` and `EXIT_FAILURE`. Platforms may additionally define their - /// own non-standard exit codes: - /// - /// | Platform | Header | - /// |-|-| - /// | macOS | [``](https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/_Exit.3.html), [``](https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/sysexits.3.html) | - /// | Linux | [``](https://sourceware.org/glibc/manual/latest/html_node/Exit-Status.html), `` | - /// | FreeBSD | [``](https://man.freebsd.org/cgi/man.cgi?exit(3)), [``](https://man.freebsd.org/cgi/man.cgi?sysexits(3)) | - /// | OpenBSD | [``](https://man.openbsd.org/exit.3), [``](https://man.openbsd.org/sysexits.3) | - /// | Windows | [``](https://learn.microsoft.com/en-us/cpp/c-runtime-library/exit-success-exit-failure) | - /// - /// On macOS, FreeBSD, OpenBSD, and Windows, the full exit code reported by - /// the process is yielded to the parent process. Linux and other POSIX-like - /// systems may only reliably report the low unsigned 8 bits (0–255) of - /// the exit code. - case exitCode(_ exitCode: CInt) - - /// The process terminated with the given signal. - /// - /// - Parameters: - /// - signal: The signal that terminated the process. - /// - /// The C programming language defines a number of [standard signals](https://en.cppreference.com/w/c/program/SIG_types). - /// Platforms may additionally define their own non-standard signal codes: - /// - /// | Platform | Header | - /// |-|-| - /// | macOS | [``](https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/signal.3.html) | - /// | Linux | [``](https://sourceware.org/glibc/manual/latest/html_node/Standard-Signals.html) | - /// | FreeBSD | [``](https://man.freebsd.org/cgi/man.cgi?signal(3)) | - /// | OpenBSD | [``](https://man.openbsd.org/signal.3) | - /// | Windows | [``](https://learn.microsoft.com/en-us/cpp/c-runtime-library/signal-constants) | - case signal(_ signal: CInt) -} - -// MARK: - Equatable - -@_spi(Experimental) -#if SWT_NO_PROCESS_SPAWNING -@available(*, unavailable, message: "Exit tests are not available on this platform.") -#endif -extension Optional { - /// Check whether or not two exit conditions are equal. - /// - /// - Parameters: - /// - lhs: One value to compare. - /// - rhs: Another value to compare. - /// - /// - Returns: Whether or not `lhs` and `rhs` are equal. - /// - /// Two exit conditions can be compared; if either instance is equal to - /// ``ExitCondition/failure``, it will compare equal to any instance except - /// ``ExitCondition/success``. To check if two instances are _exactly_ equal, - /// use the ``===(_:_:)`` operator: - /// - /// ```swift - /// let lhs: ExitCondition = .failure - /// let rhs: ExitCondition = .signal(SIGINT) - /// print(lhs == rhs) // prints "true" - /// print(lhs === rhs) // prints "false" - /// ``` - /// - /// This special behavior means that the ``==(_:_:)`` operator is not - /// transitive, and does not satisfy the requirements of - /// [`Equatable`](https://developer.apple.com/documentation/swift/equatable) - /// or [`Hashable`](https://developer.apple.com/documentation/swift/hashable). - /// - /// For any values `a` and `b`, `a == b` implies that `a != b` is `false`. - public static func ==(lhs: Self, rhs: Self) -> Bool { -#if !SWT_NO_PROCESS_SPAWNING - return switch (lhs, rhs) { - case let (.failure, .exitCode(exitCode)), let (.exitCode(exitCode), .failure): - exitCode != EXIT_SUCCESS - case (.failure, .signal), (.signal, .failure): - // All terminating signals are considered failures. - true - default: - lhs === rhs - } -#else - fatalError("Unsupported") -#endif - } - - /// Check whether or not two exit conditions are _not_ equal. - /// - /// - Parameters: - /// - lhs: One value to compare. - /// - rhs: Another value to compare. - /// - /// - Returns: Whether or not `lhs` and `rhs` are _not_ equal. - /// - /// Two exit conditions can be compared; if either instance is equal to - /// ``ExitCondition/failure``, it will compare equal to any instance except - /// ``ExitCondition/success``. To check if two instances are not _exactly_ - /// equal, use the ``!==(_:_:)`` operator: - /// - /// ```swift - /// let lhs: ExitCondition = .failure - /// let rhs: ExitCondition = .signal(SIGINT) - /// print(lhs != rhs) // prints "false" - /// print(lhs !== rhs) // prints "true" - /// ``` - /// - /// This special behavior means that the ``!=(_:_:)`` operator is not - /// transitive, and does not satisfy the requirements of - /// [`Equatable`](https://developer.apple.com/documentation/swift/equatable) - /// or [`Hashable`](https://developer.apple.com/documentation/swift/hashable). - /// - /// For any values `a` and `b`, `a == b` implies that `a != b` is `false`. - public static func !=(lhs: Self, rhs: Self) -> Bool { -#if !SWT_NO_PROCESS_SPAWNING - !(lhs == rhs) -#else - fatalError("Unsupported") -#endif - } - - /// Check whether or not two exit conditions are identical. - /// - /// - Parameters: - /// - lhs: One value to compare. - /// - rhs: Another value to compare. - /// - /// - Returns: Whether or not `lhs` and `rhs` are identical. - /// - /// Two exit conditions can be compared; if either instance is equal to - /// ``ExitCondition/failure``, it will compare equal to any instance except - /// ``ExitCondition/success``. To check if two instances are _exactly_ equal, - /// use the ``===(_:_:)`` operator: - /// - /// ```swift - /// let lhs: ExitCondition = .failure - /// let rhs: ExitCondition = .signal(SIGINT) - /// print(lhs == rhs) // prints "true" - /// print(lhs === rhs) // prints "false" - /// ``` - /// - /// This special behavior means that the ``==(_:_:)`` operator is not - /// transitive, and does not satisfy the requirements of - /// [`Equatable`](https://developer.apple.com/documentation/swift/equatable) - /// or [`Hashable`](https://developer.apple.com/documentation/swift/hashable). - /// - /// For any values `a` and `b`, `a === b` implies that `a !== b` is `false`. - public static func ===(lhs: Self, rhs: Self) -> Bool { - return switch (lhs, rhs) { - case (.none, .none): - true - case (.failure, .failure): - true - case let (.exitCode(lhs), .exitCode(rhs)): - lhs == rhs - case let (.signal(lhs), .signal(rhs)): - lhs == rhs - default: - false - } - } - - /// Check whether or not two exit conditions are _not_ identical. - /// - /// - Parameters: - /// - lhs: One value to compare. - /// - rhs: Another value to compare. - /// - /// - Returns: Whether or not `lhs` and `rhs` are _not_ identical. - /// - /// Two exit conditions can be compared; if either instance is equal to - /// ``ExitCondition/failure``, it will compare equal to any instance except - /// ``ExitCondition/success``. To check if two instances are not _exactly_ - /// equal, use the ``!==(_:_:)`` operator: - /// - /// ```swift - /// let lhs: ExitCondition = .failure - /// let rhs: ExitCondition = .signal(SIGINT) - /// print(lhs != rhs) // prints "false" - /// print(lhs !== rhs) // prints "true" - /// ``` - /// - /// This special behavior means that the ``!=(_:_:)`` operator is not - /// transitive, and does not satisfy the requirements of - /// [`Equatable`](https://developer.apple.com/documentation/swift/equatable) - /// or [`Hashable`](https://developer.apple.com/documentation/swift/hashable). - /// - /// For any values `a` and `b`, `a === b` implies that `a !== b` is `false`. - public static func !==(lhs: Self, rhs: Self) -> Bool { -#if !SWT_NO_PROCESS_SPAWNING - !(lhs === rhs) -#else - fatalError("Unsupported") -#endif - } -} diff --git a/Sources/Testing/ExitTests/ExitTest.Condition.swift b/Sources/Testing/ExitTests/ExitTest.Condition.swift new file mode 100644 index 000000000..68469280f --- /dev/null +++ b/Sources/Testing/ExitTests/ExitTest.Condition.swift @@ -0,0 +1,148 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024–2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +private import _TestingInternals + +@_spi(Experimental) +#if SWT_NO_EXIT_TESTS +@available(*, unavailable, message: "Exit tests are not available on this platform.") +#endif +extension ExitTest { + /// The possible conditions under which an exit test will complete. + /// + /// Values of this type are used to describe the conditions under which an + /// exit test is expected to pass or fail by passing them to + /// ``expect(exitsWith:observing:_:sourceLocation:performing:)`` or + /// ``require(exitsWith:observing:_:sourceLocation:performing:)``. + public struct Condition: Sendable { + /// An enumeration describing the possible requirements for an exit test. + private enum _Kind: Sendable, Equatable { + /// The exit test must exit with a particular exit status. + case statusAtExit(StatusAtExit) + + /// The exit test must exit with any failure. + case failure + } + + /// The kind of requirement. + private var _kind: _Kind + } +} + +// MARK: - + +@_spi(Experimental) +#if SWT_NO_EXIT_TESTS +@available(*, unavailable, message: "Exit tests are not available on this platform.") +#endif +extension ExitTest.Condition { + /// The process terminated successfully with exit code `EXIT_SUCCESS`. + public static var success: Self { + // Strictly speaking, the C standard treats 0 as a successful exit code and + // potentially distinct from EXIT_SUCCESS. To my knowledge, no modern + // operating system defines EXIT_SUCCESS to any value other than 0, so the + // distinction is academic. +#if !SWT_NO_EXIT_TESTS + .exitCode(EXIT_SUCCESS) +#else + fatalError("Unsupported") +#endif + } + + /// The process terminated abnormally with any exit code other than + /// `EXIT_SUCCESS` or with any signal. + public static var failure: Self { + Self(_kind: .failure) + } + + public init(_ statusAtExit: StatusAtExit) { + self.init(_kind: .statusAtExit(statusAtExit)) + } + + /// The process terminated with the given exit code. + /// + /// - Parameters: + /// - exitCode: The exit code yielded by the process. + /// + /// The C programming language defines two [standard exit codes](https://en.cppreference.com/w/c/program/EXIT_status), + /// `EXIT_SUCCESS` and `EXIT_FAILURE`. Platforms may additionally define their + /// own non-standard exit codes: + /// + /// | Platform | Header | + /// |-|-| + /// | macOS | [``](https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/_Exit.3.html), [``](https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/sysexits.3.html) | + /// | Linux | [``](https://sourceware.org/glibc/manual/latest/html_node/Exit-Status.html), `` | + /// | FreeBSD | [``](https://man.freebsd.org/cgi/man.cgi?exit(3)), [``](https://man.freebsd.org/cgi/man.cgi?sysexits(3)) | + /// | OpenBSD | [``](https://man.openbsd.org/exit.3), [``](https://man.openbsd.org/sysexits.3) | + /// | Windows | [``](https://learn.microsoft.com/en-us/cpp/c-runtime-library/exit-success-exit-failure) | + /// + /// On macOS, FreeBSD, OpenBSD, and Windows, the full exit code reported by + /// the process is yielded to the parent process. Linux and other POSIX-like + /// systems may only reliably report the low unsigned 8 bits (0–255) of + /// the exit code. + public static func exitCode(_ exitCode: CInt) -> Self { +#if !SWT_NO_EXIT_TESTS + Self(.exitCode(exitCode)) +#else + fatalError("Unsupported") +#endif + } + + /// The process terminated with the given signal. + /// + /// - Parameters: + /// - signal: The signal that terminated the process. + /// + /// The C programming language defines a number of [standard signals](https://en.cppreference.com/w/c/program/SIG_types). + /// Platforms may additionally define their own non-standard signal codes: + /// + /// | Platform | Header | + /// |-|-| + /// | macOS | [``](https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/signal.3.html) | + /// | Linux | [``](https://sourceware.org/glibc/manual/latest/html_node/Standard-Signals.html) | + /// | FreeBSD | [``](https://man.freebsd.org/cgi/man.cgi?signal(3)) | + /// | OpenBSD | [``](https://man.openbsd.org/signal.3) | + /// | Windows | [``](https://learn.microsoft.com/en-us/cpp/c-runtime-library/signal-constants) | + public static func signal(_ signal: CInt) -> Self { +#if !SWT_NO_EXIT_TESTS + Self(.signal(signal)) +#else + fatalError("Unsupported") +#endif + } +} + +// MARK: - Comparison + +#if SWT_NO_EXIT_TESTS +@available(*, unavailable, message: "Exit tests are not available on this platform.") +#endif +extension ExitTest.Condition { + /// Check whether or not an exit test condition matches a given exit status. + /// + /// - Parameters: + /// - other: Another value to compare. + /// + /// - Returns: Whether or not `self` and `other` are equal. + /// + /// Two exit test requirements can be compared; if either instance is equal to + /// ``failure``, it will compare equal to any instance except ``success``. + func isApproximatelyEqual(to statusAtExit: StatusAtExit) -> Bool { + return switch (self._kind, statusAtExit) { + case let (.failure, .exitCode(exitCode)): + exitCode != EXIT_SUCCESS + case (.failure, .signal): + // All terminating signals are considered failures. + true + default: + self._kind == .statusAtExit(statusAtExit) + } + } +} diff --git a/Sources/Testing/ExitTests/ExitTest.Result.swift b/Sources/Testing/ExitTests/ExitTest.Result.swift new file mode 100644 index 000000000..6dbf5b5d6 --- /dev/null +++ b/Sources/Testing/ExitTests/ExitTest.Result.swift @@ -0,0 +1,86 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024–2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +@_spi(Experimental) +#if SWT_NO_EXIT_TESTS +@available(*, unavailable, message: "Exit tests are not available on this platform.") +#endif +extension ExitTest { + /// A type representing the result of an exit test after it has exited and + /// returned control to the calling test function. + /// + /// Both ``expect(exitsWith:observing:_:sourceLocation:performing:)`` and + /// ``require(exitsWith:observing:_:sourceLocation:performing:)`` return + /// instances of this type. + public struct Result: Sendable { + /// The exit condition the exit test exited with. + /// + /// When the exit test passes, the value of this property is equal to the + /// exit status reported by the process that hosted the exit test. + public var statusAtExit: StatusAtExit + + /// All bytes written to the standard output stream of the exit test before + /// it exited. + /// + /// The value of this property may contain any arbitrary sequence of bytes, + /// including sequences that are not valid UTF-8 and cannot be decoded by + /// [`String.init(cString:)`](https://developer.apple.com/documentation/swift/string/init(cstring:)-6kr8s). + /// Consider using [`String.init(validatingCString:)`](https://developer.apple.com/documentation/swift/string/init(validatingcstring:)-992vo) + /// instead. + /// + /// When checking the value of this property, keep in mind that the standard + /// output stream is globally accessible, and any code running in an exit + /// test may write to it including including the operating system and any + /// third-party dependencies you have declared in your package. Rather than + /// comparing the value of this property with [`==`](https://developer.apple.com/documentation/swift/array/==(_:_:)), + /// use [`contains(_:)`](https://developer.apple.com/documentation/swift/collection/contains(_:)) + /// to check if expected output is present. + /// + /// To enable gathering output from the standard output stream during an + /// exit test, pass `\.standardOutputContent` in the `observedValues` + /// argument of ``expect(exitsWith:observing:_:sourceLocation:performing:)`` + /// or ``require(exitsWith:observing:_:sourceLocation:performing:)``. + /// + /// If you did not request standard output content when running an exit + /// test, the value of this property is the empty array. + public var standardOutputContent: [UInt8] = [] + + /// All bytes written to the standard error stream of the exit test before + /// it exited. + /// + /// The value of this property may contain any arbitrary sequence of bytes, + /// including sequences that are not valid UTF-8 and cannot be decoded by + /// [`String.init(cString:)`](https://developer.apple.com/documentation/swift/string/init(cstring:)-6kr8s). + /// Consider using [`String.init(validatingCString:)`](https://developer.apple.com/documentation/swift/string/init(validatingcstring:)-992vo) + /// instead. + /// + /// When checking the value of this property, keep in mind that the standard + /// output stream is globally accessible, and any code running in an exit + /// test may write to it including including the operating system and any + /// third-party dependencies you have declared in your package. Rather than + /// comparing the value of this property with [`==`](https://developer.apple.com/documentation/swift/array/==(_:_:)), + /// use [`contains(_:)`](https://developer.apple.com/documentation/swift/collection/contains(_:)) + /// to check if expected output is present. + /// + /// To enable gathering output from the standard error stream during an exit + /// test, pass `\.standardErrorContent` in the `observedValues` argument of + /// ``expect(exitsWith:observing:_:sourceLocation:performing:)`` or + /// ``require(exitsWith:observing:_:sourceLocation:performing:)``. + /// + /// If you did not request standard error content when running an exit test, + /// the value of this property is the empty array. + public var standardErrorContent: [UInt8] = [] + + @_spi(ForToolsIntegrationOnly) + public init(statusAtExit: StatusAtExit) { + self.statusAtExit = statusAtExit + } + } +} diff --git a/Sources/Testing/ExitTests/ExitTest.swift b/Sources/Testing/ExitTests/ExitTest.swift index 75102abe2..67bd58545 100644 --- a/Sources/Testing/ExitTests/ExitTest.swift +++ b/Sources/Testing/ExitTests/ExitTest.swift @@ -1,7 +1,7 @@ // // This source file is part of the Swift.org open source project // -// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Copyright (c) 2024–2025 Apple Inc. and the Swift project authors // Licensed under Apache License v2.0 with Runtime Library Exception // // See https://swift.org/LICENSE.txt for license information @@ -62,8 +62,8 @@ public struct ExitTest: Sendable, ~Copyable { /// /// Key paths are not sendable because the properties they refer to may or may /// not be, so this property needs to be `nonisolated(unsafe)`. It is safe to - /// use it in this fashion because `ExitTestArtifacts` is sendable. - fileprivate var _observedValues = [any PartialKeyPath & Sendable]() + /// use it in this fashion because ``ExitTest/Result`` is sendable. + fileprivate var _observedValues = [any PartialKeyPath & Sendable]() /// Key paths representing results from within this exit test that should be /// observed and returned to the caller. @@ -74,17 +74,17 @@ public struct ExitTest: Sendable, ~Copyable { /// this property to determine what information you need to preserve from your /// child process. /// - /// The value of this property always includes ``ExitTestArtifacts/exitCondition`` + /// The value of this property always includes ``ExitTest/Result/statusAtExit`` /// even if the test author does not specify it. /// /// Within a child process running an exit test, the value of this property is /// otherwise unspecified. @_spi(ForToolsIntegrationOnly) - public var observedValues: [any PartialKeyPath & Sendable] { + public var observedValues: [any PartialKeyPath & Sendable] { get { var result = _observedValues - if !result.contains(\.exitCondition) { // O(n), but n <= 3 (no Set needed) - result.append(\.exitCondition) + if !result.contains(\.statusAtExit) { // O(n), but n <= 3 (no Set needed) + result.append(\.statusAtExit) } return result } @@ -283,7 +283,7 @@ extension ExitTest { /// - expectedExitCondition: The expected exit condition. /// - observedValues: An array of key paths representing results from within /// the exit test that should be observed and returned by this macro. The -/// ``ExitTestArtifacts/exitCondition`` property is always returned. +/// ``ExitTest/Result/statusAtExit`` property is always returned. /// - expression: The expression, corresponding to `condition`, that is being /// evaluated (if available at compile time.) /// - comments: An array of comments describing the expectation. This array @@ -299,19 +299,19 @@ extension ExitTest { /// convention. func callExitTest( identifiedBy exitTestID: (UInt64, UInt64), - exitsWith expectedExitCondition: ExitCondition, - observing observedValues: [any PartialKeyPath & Sendable], + exitsWith expectedExitCondition: ExitTest.Condition, + observing observedValues: [any PartialKeyPath & Sendable], expression: __Expression, comments: @autoclosure () -> [Comment], isRequired: Bool, isolation: isolated (any Actor)? = #isolation, sourceLocation: SourceLocation -) async -> Result { +) async -> Result { guard let configuration = Configuration.current ?? Configuration.all.first else { preconditionFailure("A test must be running on the current task to use #expect(exitsWith:).") } - var result: ExitTestArtifacts + var result: ExitTest.Result do { var exitTest = ExitTest(id: ExitTest.ID(exitTestID)) exitTest.observedValues = observedValues @@ -320,8 +320,8 @@ func callExitTest( #if os(Windows) // For an explanation of this magic, see the corresponding logic in // ExitTest.callAsFunction(). - if case let .exitCode(exitCode) = result.exitCondition, (exitCode & ~STATUS_CODE_MASK) == STATUS_SIGNAL_CAUGHT_BITS { - result.exitCondition = .signal(exitCode & STATUS_CODE_MASK) + if case let .exitCode(exitCode) = result.statusAtExit, (exitCode & ~STATUS_CODE_MASK) == STATUS_SIGNAL_CAUGHT_BITS { + result.statusAtExit = .signal(exitCode & STATUS_CODE_MASK) } #endif } catch { @@ -346,17 +346,22 @@ func callExitTest( // For lack of a better way to handle an exit test failing in this way, // we record the system issue above, then let the expectation fail below by // reporting an exit condition that's the inverse of the expected one. - result = ExitTestArtifacts(exitCondition: expectedExitCondition == .failure ? .success : .failure) + let statusAtExit: StatusAtExit = if expectedExitCondition.isApproximatelyEqual(to: .exitCode(EXIT_FAILURE)) { + .exitCode(EXIT_SUCCESS) + } else { + .exitCode(EXIT_FAILURE) + } + result = ExitTest.Result(statusAtExit: statusAtExit) } // How did the exit test actually exit? - let actualExitCondition = result.exitCondition + let actualStatusAtExit = result.statusAtExit // Plumb the exit test's result through the general expectation machinery. return __checkValue( - expectedExitCondition == actualExitCondition, + expectedExitCondition.isApproximatelyEqual(to: actualStatusAtExit), expression: expression, - expressionWithCapturedRuntimeValues: expression.capturingRuntimeValues(actualExitCondition), + expressionWithCapturedRuntimeValues: expression.capturingRuntimeValues(actualStatusAtExit), mismatchedExitConditionDescription: String(describingForTest: expectedExitCondition), comments: comments(), isRequired: isRequired, @@ -404,7 +409,7 @@ extension ExitTest { /// are available or the child environment is otherwise terminated. The parent /// environment is then responsible for interpreting those results and /// recording any issues that occur. - public typealias Handler = @Sendable (_ exitTest: borrowing ExitTest) async throws -> ExitTestArtifacts + public typealias Handler = @Sendable (_ exitTest: borrowing ExitTest) async throws -> ExitTest.Result /// The back channel file handle set up by the parent process. /// @@ -581,7 +586,7 @@ extension ExitTest { childEnvironment["SWT_EXPERIMENTAL_EXIT_TEST_ID"] = String(decoding: json, as: UTF8.self) } - typealias ResultUpdater = @Sendable (inout ExitTestArtifacts) -> Void + typealias ResultUpdater = @Sendable (inout ExitTest.Result) -> Void return try await withThrowingTaskGroup(of: ResultUpdater?.self) { taskGroup in // Set up stdout and stderr streams. By POSIX convention, stdin/stdout // are line-buffered by default and stderr is unbuffered by default. @@ -641,8 +646,8 @@ extension ExitTest { // Await termination of the child process. taskGroup.addTask { - let exitCondition = try await wait(for: processID) - return { $0.exitCondition = exitCondition } + let statusAtExit = try await wait(for: processID) + return { $0.statusAtExit = statusAtExit } } // Read back the stdout and stderr streams. @@ -669,10 +674,10 @@ extension ExitTest { return nil } - // Collate the various bits of the result. The exit condition .failure - // here is just a placeholder and will be replaced by the result of one - // of the tasks above. - var result = ExitTestArtifacts(exitCondition: .failure) + // Collate the various bits of the result. The exit condition used here + // is just a placeholder and will be replaced by the result of one of + // the tasks above. + var result = ExitTest.Result(statusAtExit: .exitCode(EXIT_FAILURE)) for try await update in taskGroup { update?(&result) } diff --git a/Sources/Testing/ExitTests/ExitTestArtifacts.swift b/Sources/Testing/ExitTests/ExitTestArtifacts.swift deleted file mode 100644 index 6e1710e2a..000000000 --- a/Sources/Testing/ExitTests/ExitTestArtifacts.swift +++ /dev/null @@ -1,90 +0,0 @@ -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2024 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for Swift project authors -// - -/// A type representing the result of an exit test after it has exited and -/// returned control to the calling test function. -/// -/// Both ``expect(exitsWith:observing:_:sourceLocation:performing:)`` and -/// ``require(exitsWith:observing:_:sourceLocation:performing:)`` return -/// instances of this type. -/// -/// - Warning: The name of this type is still unstable and subject to change. -@_spi(Experimental) -#if SWT_NO_EXIT_TESTS -@available(*, unavailable, message: "Exit tests are not available on this platform.") -#endif -public struct ExitTestArtifacts: Sendable { - /// The exit condition the exit test exited with. - /// - /// When the exit test passes, the value of this property is equal to the - /// value of the `expectedExitCondition` argument passed to - /// ``expect(exitsWith:observing:_:sourceLocation:performing:)`` or to - /// ``require(exitsWith:observing:_:sourceLocation:performing:)``. You can - /// compare two instances of ``ExitCondition`` with - /// ``/Swift/Optional/==(_:_:)``. - public var exitCondition: ExitCondition - - /// All bytes written to the standard output stream of the exit test before - /// it exited. - /// - /// The value of this property may contain any arbitrary sequence of bytes, - /// including sequences that are not valid UTF-8 and cannot be decoded by - /// [`String.init(cString:)`](https://developer.apple.com/documentation/swift/string/init(cstring:)-6kr8s). - /// Consider using [`String.init(validatingCString:)`](https://developer.apple.com/documentation/swift/string/init(validatingcstring:)-992vo) - /// instead. - /// - /// When checking the value of this property, keep in mind that the standard - /// output stream is globally accessible, and any code running in an exit - /// test may write to it including including the operating system and any - /// third-party dependencies you have declared in your package. Rather than - /// comparing the value of this property with [`==`](https://developer.apple.com/documentation/swift/array/==(_:_:)), - /// use [`contains(_:)`](https://developer.apple.com/documentation/swift/collection/contains(_:)) - /// to check if expected output is present. - /// - /// To enable gathering output from the standard output stream during an - /// exit test, pass `\.standardOutputContent` in the `observedValues` - /// argument of ``expect(exitsWith:observing:_:sourceLocation:performing:)`` - /// or ``require(exitsWith:observing:_:sourceLocation:performing:)``. - /// - /// If you did not request standard output content when running an exit test, - /// the value of this property is the empty array. - public var standardOutputContent: [UInt8] = [] - - /// All bytes written to the standard error stream of the exit test before - /// it exited. - /// - /// The value of this property may contain any arbitrary sequence of bytes, - /// including sequences that are not valid UTF-8 and cannot be decoded by - /// [`String.init(cString:)`](https://developer.apple.com/documentation/swift/string/init(cstring:)-6kr8s). - /// Consider using [`String.init(validatingCString:)`](https://developer.apple.com/documentation/swift/string/init(validatingcstring:)-992vo) - /// instead. - /// - /// When checking the value of this property, keep in mind that the standard - /// output stream is globally accessible, and any code running in an exit - /// test may write to it including including the operating system and any - /// third-party dependencies you have declared in your package. Rather than - /// comparing the value of this property with [`==`](https://developer.apple.com/documentation/swift/array/==(_:_:)), - /// use [`contains(_:)`](https://developer.apple.com/documentation/swift/collection/contains(_:)) - /// to check if expected output is present. - /// - /// To enable gathering output from the standard error stream during an exit - /// test, pass `\.standardErrorContent` in the `observedValues` argument of - /// ``expect(exitsWith:observing:_:sourceLocation:performing:)`` or - /// ``require(exitsWith:observing:_:sourceLocation:performing:)``. - /// - /// If you did not request standard error content when running an exit test, - /// the value of this property is the empty array. - public var standardErrorContent: [UInt8] = [] - - @_spi(ForToolsIntegrationOnly) - public init(exitCondition: ExitCondition) { - self.exitCondition = exitCondition - } -} diff --git a/Sources/Testing/ExitTests/SpawnProcess.swift b/Sources/Testing/ExitTests/SpawnProcess.swift index 4726e8f41..c824baa4e 100644 --- a/Sources/Testing/ExitTests/SpawnProcess.swift +++ b/Sources/Testing/ExitTests/SpawnProcess.swift @@ -1,7 +1,7 @@ // // This source file is part of the Swift.org open source project // -// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Copyright (c) 2024–2025 Apple Inc. and the Swift project authors // Licensed under Apache License v2.0 with Runtime Library Exception // // See https://swift.org/LICENSE.txt for license information diff --git a/Sources/Testing/ExitTests/StatusAtExit.swift b/Sources/Testing/ExitTests/StatusAtExit.swift new file mode 100644 index 000000000..7817cbb52 --- /dev/null +++ b/Sources/Testing/ExitTests/StatusAtExit.swift @@ -0,0 +1,73 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024–2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +private import _TestingInternals + +/// An enumeration describing possible status a process will yield on exit. +/// +/// You can cast an instance of this type to an instance of +/// ``ExitTest/Condition`` using ``ExitTest/Condition/init(_:)``. That value +/// can then be used to describe the condition under which an exit test is +/// expected to pass or fail by passing it to +/// ``expect(exitsWith:observing:_:sourceLocation:performing:)`` or +/// ``require(exitsWith:observing:_:sourceLocation:performing:)``. +@_spi(Experimental) +#if SWT_NO_PROCESS_SPAWNING +@available(*, unavailable, message: "Exit tests are not available on this platform.") +#endif +public enum StatusAtExit: Sendable { + /// The process terminated with the given exit code. + /// + /// - Parameters: + /// - exitCode: The exit code yielded by the process. + /// + /// The C programming language defines two [standard exit codes](https://en.cppreference.com/w/c/program/EXIT_status), + /// `EXIT_SUCCESS` and `EXIT_FAILURE`. Platforms may additionally define their + /// own non-standard exit codes: + /// + /// | Platform | Header | + /// |-|-| + /// | macOS | [``](https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/_Exit.3.html), [``](https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/sysexits.3.html) | + /// | Linux | [``](https://sourceware.org/glibc/manual/latest/html_node/Exit-Status.html), `` | + /// | FreeBSD | [``](https://man.freebsd.org/cgi/man.cgi?exit(3)), [``](https://man.freebsd.org/cgi/man.cgi?sysexits(3)) | + /// | OpenBSD | [``](https://man.openbsd.org/exit.3), [``](https://man.openbsd.org/sysexits.3) | + /// | Windows | [``](https://learn.microsoft.com/en-us/cpp/c-runtime-library/exit-success-exit-failure) | + /// + /// On macOS, FreeBSD, OpenBSD, and Windows, the full exit code reported by + /// the process is yielded to the parent process. Linux and other POSIX-like + /// systems may only reliably report the low unsigned 8 bits (0–255) of + /// the exit code. + case exitCode(_ exitCode: CInt) + + /// The process terminated with the given signal. + /// + /// - Parameters: + /// - signal: The signal that terminated the process. + /// + /// The C programming language defines a number of [standard signals](https://en.cppreference.com/w/c/program/SIG_types). + /// Platforms may additionally define their own non-standard signal codes: + /// + /// | Platform | Header | + /// |-|-| + /// | macOS | [``](https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/signal.3.html) | + /// | Linux | [``](https://sourceware.org/glibc/manual/latest/html_node/Standard-Signals.html) | + /// | FreeBSD | [``](https://man.freebsd.org/cgi/man.cgi?signal(3)) | + /// | OpenBSD | [``](https://man.openbsd.org/signal.3) | + /// | Windows | [``](https://learn.microsoft.com/en-us/cpp/c-runtime-library/signal-constants) | + case signal(_ signal: CInt) +} + +// MARK: - Equatable + +@_spi(Experimental) +#if SWT_NO_PROCESS_SPAWNING +@available(*, unavailable, message: "Exit tests are not available on this platform.") +#endif +extension StatusAtExit: Equatable {} diff --git a/Sources/Testing/ExitTests/WaitFor.swift b/Sources/Testing/ExitTests/WaitFor.swift index ccd6512b6..ef910f6cb 100644 --- a/Sources/Testing/ExitTests/WaitFor.swift +++ b/Sources/Testing/ExitTests/WaitFor.swift @@ -1,7 +1,7 @@ // // This source file is part of the Swift.org open source project // -// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Copyright (c) 2024–2025 Apple Inc. and the Swift project authors // Licensed under Apache License v2.0 with Runtime Library Exception // // See https://swift.org/LICENSE.txt for license information @@ -20,7 +20,7 @@ internal import _TestingInternals /// /// - Throws: If the exit status of the process with ID `pid` cannot be /// determined (i.e. it does not represent an exit condition.) -private func _blockAndWait(for pid: consuming pid_t) throws -> ExitCondition { +private func _blockAndWait(for pid: consuming pid_t) throws -> StatusAtExit { let pid = consume pid // Get the exit status of the process or throw an error (other than EINTR.) @@ -61,7 +61,7 @@ private func _blockAndWait(for pid: consuming pid_t) throws -> ExitCondition { /// - Note: The open-source implementation of libdispatch available on Linux /// and other platforms does not support `DispatchSourceProcess`. Those /// platforms use an alternate implementation below. -func wait(for pid: consuming pid_t) async throws -> ExitCondition { +func wait(for pid: consuming pid_t) async throws -> StatusAtExit { let pid = consume pid let source = DispatchSource.makeProcessSource(identifier: pid, eventMask: .exit) @@ -80,7 +80,7 @@ func wait(for pid: consuming pid_t) async throws -> ExitCondition { } #elseif SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(OpenBSD) /// A mapping of awaited child PIDs to their corresponding Swift continuations. -private let _childProcessContinuations = LockedWith]>() +private let _childProcessContinuations = LockedWith]>() /// A condition variable used to suspend the waiter thread created by /// `_createWaitThread()` when there are no child processes to await. @@ -202,7 +202,7 @@ private let _createWaitThread: Void = { /// /// On Apple platforms, the libdispatch-based implementation above is more /// efficient because it does not need to permanently reserve a thread. -func wait(for pid: consuming pid_t) async throws -> ExitCondition { +func wait(for pid: consuming pid_t) async throws -> StatusAtExit { let pid = consume pid // Ensure the waiter thread is running. @@ -239,7 +239,7 @@ func wait(for pid: consuming pid_t) async throws -> ExitCondition { /// This implementation of `wait(for:)` calls `RegisterWaitForSingleObject()` to /// wait for `processHandle`, suspends the calling task until the waiter's /// callback is called, then calls `GetExitCodeProcess()`. -func wait(for processHandle: consuming HANDLE) async throws -> ExitCondition { +func wait(for processHandle: consuming HANDLE) async throws -> StatusAtExit { let processHandle = consume processHandle defer { _ = CloseHandle(processHandle) @@ -283,6 +283,6 @@ func wait(for processHandle: consuming HANDLE) async throws -> ExitCondition { } #else #warning("Platform-specific implementation missing: cannot wait for child processes to exit") -func wait(for processID: consuming Never) async throws -> ExitCondition {} +func wait(for processID: consuming Never) async throws -> StatusAtExit {} #endif #endif diff --git a/Sources/Testing/Expectations/Expectation+Macro.swift b/Sources/Testing/Expectations/Expectation+Macro.swift index 7e82b2144..bb6b68ab0 100644 --- a/Sources/Testing/Expectations/Expectation+Macro.swift +++ b/Sources/Testing/Expectations/Expectation+Macro.swift @@ -444,13 +444,13 @@ public macro require( /// - expectedExitCondition: The expected exit condition. /// - observedValues: An array of key paths representing results from within /// the exit test that should be observed and returned by this macro. The -/// ``ExitTestArtifacts/exitCondition`` property is always returned. +/// ``ExitTest/Result/statusAtExit`` property is always returned. /// - comment: A comment describing the expectation. /// - sourceLocation: The source location to which recorded expectations and /// issues should be attributed. /// - expression: The expression to be evaluated. /// -/// - Returns: If the exit test passes, an instance of ``ExitTestArtifacts`` +/// - Returns: If the exit test passes, an instance of ``ExitTest/Result`` /// describing the state of the exit test when it exited. If the exit test /// fails, the result is `nil`. /// @@ -483,8 +483,8 @@ public macro require( /// process is terminated. /// /// Once the child process terminates, the parent process resumes and compares -/// its exit status against `exitCondition`. If they match, the exit test has -/// passed; otherwise, it has failed and an issue is recorded. +/// its exit status against `expectedExitCondition`. If they match, the exit +/// test has passed; otherwise, it has failed and an issue is recorded. /// /// ## Child process output /// @@ -542,12 +542,12 @@ public macro require( #endif @discardableResult @freestanding(expression) public macro expect( - exitsWith expectedExitCondition: ExitCondition, - observing observedValues: [any PartialKeyPath & Sendable] = [], + exitsWith expectedExitCondition: ExitTest.Condition, + observing observedValues: [any PartialKeyPath & Sendable] = [], _ comment: @autoclosure () -> Comment? = nil, sourceLocation: SourceLocation = #_sourceLocation, performing expression: @escaping @Sendable @convention(thin) () async throws -> Void -) -> ExitTestArtifacts? = #externalMacro(module: "TestingMacros", type: "ExitTestExpectMacro") +) -> ExitTest.Result? = #externalMacro(module: "TestingMacros", type: "ExitTestExpectMacro") /// Check that an expression causes the process to terminate in a given fashion /// and throw an error if it did not. @@ -556,13 +556,13 @@ public macro require( /// - expectedExitCondition: The expected exit condition. /// - observedValues: An array of key paths representing results from within /// the exit test that should be observed and returned by this macro. The -/// ``ExitTestArtifacts/exitCondition`` property is always returned. +/// ``ExitTest/Result/statusAtExit`` property is always returned. /// - comment: A comment describing the expectation. /// - sourceLocation: The source location to which recorded expectations and /// issues should be attributed. /// - expression: The expression to be evaluated. /// -/// - Returns: An instance of ``ExitTestArtifacts`` describing the state of the +/// - Returns: An instance of ``ExitTest/Result`` describing the state of the /// exit test when it exited. /// /// - Throws: An instance of ``ExpectationFailedError`` if the exit condition of @@ -597,8 +597,8 @@ public macro require( /// process is terminated. /// /// Once the child process terminates, the parent process resumes and compares -/// its exit status against `exitCondition`. If they match, the exit test has -/// passed; otherwise, it has failed and an issue is recorded. +/// its exit status against `expectedExitCondition`. If they match, the exit +/// test has passed; otherwise, it has failed and an issue is recorded. /// /// ## Child process output /// @@ -654,9 +654,9 @@ public macro require( #endif @discardableResult @freestanding(expression) public macro require( - exitsWith expectedExitCondition: ExitCondition, - observing observedValues: [any PartialKeyPath & Sendable] = [], + exitsWith expectedExitCondition: ExitTest.Condition, + observing observedValues: [any PartialKeyPath & Sendable] = [], _ comment: @autoclosure () -> Comment? = nil, sourceLocation: SourceLocation = #_sourceLocation, performing expression: @escaping @Sendable @convention(thin) () async throws -> Void -) -> ExitTestArtifacts = #externalMacro(module: "TestingMacros", type: "ExitTestRequireMacro") +) -> ExitTest.Result = #externalMacro(module: "TestingMacros", type: "ExitTestRequireMacro") diff --git a/Sources/Testing/Expectations/ExpectationChecking+Macro.swift b/Sources/Testing/Expectations/ExpectationChecking+Macro.swift index 550f6039c..beeefa0e6 100644 --- a/Sources/Testing/Expectations/ExpectationChecking+Macro.swift +++ b/Sources/Testing/Expectations/ExpectationChecking+Macro.swift @@ -1148,18 +1148,18 @@ public func __checkClosureCall( @_spi(Experimental) public func __checkClosureCall( identifiedBy exitTestID: (UInt64, UInt64), - exitsWith expectedExitCondition: ExitCondition, - observing observedValues: [any PartialKeyPath & Sendable], + exitsWith requirement: ExitTest.Condition, + observing observedValues: [any PartialKeyPath & Sendable], performing body: @convention(thin) () -> Void, expression: __Expression, comments: @autoclosure () -> [Comment], isRequired: Bool, isolation: isolated (any Actor)? = #isolation, sourceLocation: SourceLocation -) async -> Result { +) async -> Result { await callExitTest( identifiedBy: exitTestID, - exitsWith: expectedExitCondition, + exitsWith: requirement, observing: observedValues, expression: expression, comments: comments(), diff --git a/Sources/TestingMacros/ConditionMacro.swift b/Sources/TestingMacros/ConditionMacro.swift index f687aa631..b4f5af1c3 100644 --- a/Sources/TestingMacros/ConditionMacro.swift +++ b/Sources/TestingMacros/ConditionMacro.swift @@ -415,15 +415,15 @@ extension ExitTestConditionMacro { _ = try Base.expansion(of: macro, in: context) var arguments = argumentList(of: macro, in: context) - let expectedExitConditionIndex = arguments.firstIndex { $0.label?.tokenKind == .identifier("exitsWith") } - guard let expectedExitConditionIndex else { - fatalError("Could not find the exit condition for this exit test. Please file a bug report at https://github.com/swiftlang/swift-testing/issues/new") + let requirementIndex = arguments.firstIndex { $0.label?.tokenKind == .identifier("exitsWith") } + guard let requirementIndex else { + fatalError("Could not find the requirement for this exit test. Please file a bug report at https://github.com/swiftlang/swift-testing/issues/new") } let observationListIndex = arguments.firstIndex { $0.label?.tokenKind == .identifier("observing") } if observationListIndex == nil { arguments.insert( Argument(label: "observing", expression: ArrayExprSyntax(expressions: [])), - at: arguments.index(after: expectedExitConditionIndex) + at: arguments.index(after: requirementIndex) ) } let trailingClosureIndex = arguments.firstIndex { $0.label?.tokenKind == _trailingClosureLabel.tokenKind } diff --git a/Tests/TestingTests/ExitTestTests.swift b/Tests/TestingTests/ExitTestTests.swift index 00d54f3f6..bc3425e0a 100644 --- a/Tests/TestingTests/ExitTestTests.swift +++ b/Tests/TestingTests/ExitTestTests.swift @@ -88,29 +88,15 @@ private import _TestingInternals // Mock an exit test where the process exits successfully. configuration.exitTestHandler = { _ in - return ExitTestArtifacts(exitCondition: .exitCode(EXIT_SUCCESS)) + return ExitTest.Result(statusAtExit: .exitCode(EXIT_SUCCESS)) } await Test { await #expect(exitsWith: .success) {} }.run(configuration: configuration) - // Mock an exit test where the process exits with a generic failure. - configuration.exitTestHandler = { _ in - return ExitTestArtifacts(exitCondition: .failure) - } - await Test { - await #expect(exitsWith: .failure) {} - }.run(configuration: configuration) - await Test { - await #expect(exitsWith: .exitCode(EXIT_FAILURE)) {} - }.run(configuration: configuration) - await Test { - await #expect(exitsWith: .signal(SIGABRT)) {} - }.run(configuration: configuration) - // Mock an exit test where the process exits with a particular error code. configuration.exitTestHandler = { _ in - return ExitTestArtifacts(exitCondition: .exitCode(123)) + return ExitTest.Result(statusAtExit: .exitCode(123)) } await Test { await #expect(exitsWith: .failure) {} @@ -118,7 +104,7 @@ private import _TestingInternals // Mock an exit test where the process exits with a signal. configuration.exitTestHandler = { _ in - return ExitTestArtifacts(exitCondition: .signal(SIGABRT)) + return ExitTest.Result(statusAtExit: .signal(SIGABRT)) } await Test { await #expect(exitsWith: .signal(SIGABRT)) {} @@ -140,7 +126,7 @@ private import _TestingInternals // Mock exit tests that were expected to fail but passed. configuration.exitTestHandler = { _ in - return ExitTestArtifacts(exitCondition: .exitCode(EXIT_SUCCESS)) + return ExitTest.Result(statusAtExit: .exitCode(EXIT_SUCCESS)) } await Test { await #expect(exitsWith: .failure) {} @@ -154,7 +140,7 @@ private import _TestingInternals // Mock exit tests that unexpectedly signalled. configuration.exitTestHandler = { _ in - return ExitTestArtifacts(exitCondition: .signal(SIGABRT)) + return ExitTest.Result(statusAtExit: .signal(SIGABRT)) } await Test { await #expect(exitsWith: .exitCode(EXIT_SUCCESS)) {} @@ -239,36 +225,6 @@ private import _TestingInternals } #endif - @Test("Exit condition matching operators (==, !=, ===, !==)") - func exitConditionMatching() { - #expect(Optional.none == Optional.none) - #expect(Optional.none === Optional.none) - #expect(Optional.none !== .some(.success)) - #expect(Optional.none !== .some(.failure)) - - #expect(ExitCondition.success == .success) - #expect(ExitCondition.success === .success) - #expect(ExitCondition.success == .exitCode(EXIT_SUCCESS)) - #expect(ExitCondition.success === .exitCode(EXIT_SUCCESS)) - #expect(ExitCondition.success != .exitCode(EXIT_FAILURE)) - #expect(ExitCondition.success !== .exitCode(EXIT_FAILURE)) - - #expect(ExitCondition.failure == .failure) - #expect(ExitCondition.failure === .failure) - - #expect(ExitCondition.exitCode(EXIT_FAILURE &+ 1) != .exitCode(EXIT_FAILURE)) - #expect(ExitCondition.exitCode(EXIT_FAILURE &+ 1) !== .exitCode(EXIT_FAILURE)) - - #expect(ExitCondition.success != .exitCode(EXIT_FAILURE)) - #expect(ExitCondition.success !== .exitCode(EXIT_FAILURE)) - #expect(ExitCondition.success != .signal(SIGINT)) - #expect(ExitCondition.success !== .signal(SIGINT)) - #expect(ExitCondition.signal(SIGINT) == .signal(SIGINT)) - #expect(ExitCondition.signal(SIGINT) === .signal(SIGINT)) - #expect(ExitCondition.signal(SIGTERM) != .signal(SIGINT)) - #expect(ExitCondition.signal(SIGTERM) !== .signal(SIGINT)) - } - @MainActor static func someMainActorFunction() { MainActor.assertIsolated() } @@ -289,21 +245,21 @@ private import _TestingInternals var result = await #expect(exitsWith: .success) { exit(EXIT_SUCCESS) } - #expect(result?.exitCondition === .success) + #expect(result?.statusAtExit == .exitCode(EXIT_SUCCESS)) result = await #expect(exitsWith: .exitCode(123)) { exit(123) } - #expect(result?.exitCondition === .exitCode(123)) + #expect(result?.statusAtExit == .exitCode(123)) // Test that basic passing exit tests produce the correct results (#require) result = try await #require(exitsWith: .success) { exit(EXIT_SUCCESS) } - #expect(result?.exitCondition === .success) + #expect(result?.statusAtExit == .exitCode(EXIT_SUCCESS)) result = try await #require(exitsWith: .exitCode(123)) { exit(123) } - #expect(result?.exitCondition === .exitCode(123)) + #expect(result?.statusAtExit == .exitCode(123)) } @Test("Result is nil on failure") @@ -322,7 +278,7 @@ private import _TestingInternals } } configuration.exitTestHandler = { _ in - ExitTestArtifacts(exitCondition: .exitCode(123)) + ExitTest.Result(statusAtExit: .exitCode(123)) } await Test { @@ -345,7 +301,7 @@ private import _TestingInternals } } configuration.exitTestHandler = { _ in - ExitTestArtifacts(exitCondition: .failure) + ExitTest.Result(statusAtExit: .exitCode(EXIT_FAILURE)) } await Test { @@ -392,7 +348,7 @@ private import _TestingInternals try FileHandle.stderr.write(String("STANDARD ERROR".reversed())) exit(EXIT_SUCCESS) } - #expect(result.exitCondition === .success) + #expect(result.statusAtExit == .exitCode(EXIT_SUCCESS)) #expect(result.standardOutputContent.contains("STANDARD OUTPUT".utf8)) #expect(result.standardErrorContent.isEmpty) @@ -401,7 +357,7 @@ private import _TestingInternals try FileHandle.stderr.write(String("STANDARD ERROR".reversed())) exit(EXIT_SUCCESS) } - #expect(result.exitCondition === .success) + #expect(result.statusAtExit == .exitCode(EXIT_SUCCESS)) #expect(result.standardOutputContent.isEmpty) #expect(result.standardErrorContent.contains("STANDARD ERROR".utf8.reversed())) } @@ -409,7 +365,7 @@ private import _TestingInternals @Test("Arguments to the macro are not captured during expansion (do not need to be literals/const)") func argumentsAreNotCapturedDuringMacroExpansion() async throws { let unrelatedSourceLocation = #_sourceLocation - func nonConstExitCondition() async throws -> ExitCondition { + func nonConstExitCondition() async throws -> ExitTest.Condition { .failure } await #expect(exitsWith: try await nonConstExitCondition(), sourceLocation: unrelatedSourceLocation) { From a87221296976146b8d8766edd6b092b20fb30118 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Wed, 26 Feb 2025 09:03:32 -0500 Subject: [PATCH 2/4] Missed a Windows-specific .failure usage --- Sources/Testing/ExitTests/WaitFor.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/Testing/ExitTests/WaitFor.swift b/Sources/Testing/ExitTests/WaitFor.swift index ef910f6cb..cc611158f 100644 --- a/Sources/Testing/ExitTests/WaitFor.swift +++ b/Sources/Testing/ExitTests/WaitFor.swift @@ -276,7 +276,7 @@ func wait(for processHandle: consuming HANDLE) async throws -> StatusAtExit { guard GetExitCodeProcess(processHandle, &status) else { // The child process terminated but we couldn't get its status back. // Assume generic failure. - return .failure + return .exitCode(EXIT_FAILURE) } return .exitCode(CInt(bitPattern: .init(status))) From a265536f2e3abb9ddb233f440ae9666e10b48c18 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Wed, 26 Feb 2025 10:36:56 -0500 Subject: [PATCH 3/4] Fix typo in DocC comment --- Sources/Testing/ExitTests/ExitTest.swift | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Sources/Testing/ExitTests/ExitTest.swift b/Sources/Testing/ExitTests/ExitTest.swift index 67bd58545..9425de432 100644 --- a/Sources/Testing/ExitTests/ExitTest.swift +++ b/Sources/Testing/ExitTests/ExitTest.swift @@ -25,9 +25,9 @@ private import _TestingInternals /// A type describing an exit test. /// /// Instances of this type describe exit tests you create using the -/// ``expect(exitsWith:_:sourceLocation:performing:)`` or -/// ``require(exitsWith:_:sourceLocation:performing:)`` macro. You don't usually -/// need to interact directly with an instance of this type. +/// ``expect(exitsWith:observing:_:sourceLocation:performing:)`` +/// ``require(exitsWith:observing:_:sourceLocation:performing:)`` macro. You +/// don't usually need to interact directly with an instance of this type. @_spi(Experimental) #if SWT_NO_EXIT_TESTS @available(*, unavailable, message: "Exit tests are not available on this platform.") From ae3939d3df7f622d6161648fc9de19e36807f428 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Thu, 27 Feb 2025 13:30:50 -0500 Subject: [PATCH 4/4] Incorporate feedback --- .../ExitTests/ExitTest.Condition.swift | 24 +++++++++++-------- .../Testing/ExitTests/ExitTest.Result.swift | 2 +- Sources/Testing/ExitTests/StatusAtExit.swift | 2 +- .../ExpectationChecking+Macro.swift | 4 ++-- 4 files changed, 18 insertions(+), 14 deletions(-) diff --git a/Sources/Testing/ExitTests/ExitTest.Condition.swift b/Sources/Testing/ExitTests/ExitTest.Condition.swift index 68469280f..10f2a6ff0 100644 --- a/Sources/Testing/ExitTests/ExitTest.Condition.swift +++ b/Sources/Testing/ExitTests/ExitTest.Condition.swift @@ -22,7 +22,7 @@ extension ExitTest { /// ``expect(exitsWith:observing:_:sourceLocation:performing:)`` or /// ``require(exitsWith:observing:_:sourceLocation:performing:)``. public struct Condition: Sendable { - /// An enumeration describing the possible requirements for an exit test. + /// An enumeration describing the possible conditions for an exit test. private enum _Kind: Sendable, Equatable { /// The exit test must exit with a particular exit status. case statusAtExit(StatusAtExit) @@ -31,7 +31,7 @@ extension ExitTest { case failure } - /// The kind of requirement. + /// The kind of condition. private var _kind: _Kind } } @@ -43,7 +43,8 @@ extension ExitTest { @available(*, unavailable, message: "Exit tests are not available on this platform.") #endif extension ExitTest.Condition { - /// The process terminated successfully with exit code `EXIT_SUCCESS`. + /// A condition that matches when a process terminates successfully with exit + /// code `EXIT_SUCCESS`. public static var success: Self { // Strictly speaking, the C standard treats 0 as a successful exit code and // potentially distinct from EXIT_SUCCESS. To my knowledge, no modern @@ -56,8 +57,8 @@ extension ExitTest.Condition { #endif } - /// The process terminated abnormally with any exit code other than - /// `EXIT_SUCCESS` or with any signal. + /// A condition that matches when a process terminates abnormally with any + /// exit code other than `EXIT_SUCCESS` or with any signal. public static var failure: Self { Self(_kind: .failure) } @@ -66,7 +67,8 @@ extension ExitTest.Condition { self.init(_kind: .statusAtExit(statusAtExit)) } - /// The process terminated with the given exit code. + /// Creates a condition that matches when a process terminates with a given + /// exit code. /// /// - Parameters: /// - exitCode: The exit code yielded by the process. @@ -95,7 +97,8 @@ extension ExitTest.Condition { #endif } - /// The process terminated with the given signal. + /// Creates a condition that matches when a process terminates with a given + /// signal. /// /// - Parameters: /// - signal: The signal that terminated the process. @@ -128,11 +131,12 @@ extension ExitTest.Condition { /// Check whether or not an exit test condition matches a given exit status. /// /// - Parameters: - /// - other: Another value to compare. + /// - statusAtExit: An exit status to compare against. /// - /// - Returns: Whether or not `self` and `other` are equal. + /// - Returns: Whether or not `self` and `statusAtExit` represent the same + /// exit condition. /// - /// Two exit test requirements can be compared; if either instance is equal to + /// Two exit test conditions can be compared; if either instance is equal to /// ``failure``, it will compare equal to any instance except ``success``. func isApproximatelyEqual(to statusAtExit: StatusAtExit) -> Bool { return switch (self._kind, statusAtExit) { diff --git a/Sources/Testing/ExitTests/ExitTest.Result.swift b/Sources/Testing/ExitTests/ExitTest.Result.swift index 6dbf5b5d6..beb2d56fc 100644 --- a/Sources/Testing/ExitTests/ExitTest.Result.swift +++ b/Sources/Testing/ExitTests/ExitTest.Result.swift @@ -62,7 +62,7 @@ extension ExitTest { /// instead. /// /// When checking the value of this property, keep in mind that the standard - /// output stream is globally accessible, and any code running in an exit + /// error stream is globally accessible, and any code running in an exit /// test may write to it including including the operating system and any /// third-party dependencies you have declared in your package. Rather than /// comparing the value of this property with [`==`](https://developer.apple.com/documentation/swift/array/==(_:_:)), diff --git a/Sources/Testing/ExitTests/StatusAtExit.swift b/Sources/Testing/ExitTests/StatusAtExit.swift index 7817cbb52..26514ffa5 100644 --- a/Sources/Testing/ExitTests/StatusAtExit.swift +++ b/Sources/Testing/ExitTests/StatusAtExit.swift @@ -12,7 +12,7 @@ private import _TestingInternals /// An enumeration describing possible status a process will yield on exit. /// -/// You can cast an instance of this type to an instance of +/// You can convert an instance of this type to an instance of /// ``ExitTest/Condition`` using ``ExitTest/Condition/init(_:)``. That value /// can then be used to describe the condition under which an exit test is /// expected to pass or fail by passing it to diff --git a/Sources/Testing/Expectations/ExpectationChecking+Macro.swift b/Sources/Testing/Expectations/ExpectationChecking+Macro.swift index beeefa0e6..7254ad049 100644 --- a/Sources/Testing/Expectations/ExpectationChecking+Macro.swift +++ b/Sources/Testing/Expectations/ExpectationChecking+Macro.swift @@ -1148,7 +1148,7 @@ public func __checkClosureCall( @_spi(Experimental) public func __checkClosureCall( identifiedBy exitTestID: (UInt64, UInt64), - exitsWith requirement: ExitTest.Condition, + exitsWith expectedExitCondition: ExitTest.Condition, observing observedValues: [any PartialKeyPath & Sendable], performing body: @convention(thin) () -> Void, expression: __Expression, @@ -1159,7 +1159,7 @@ public func __checkClosureCall( ) async -> Result { await callExitTest( identifiedBy: exitTestID, - exitsWith: requirement, + exitsWith: expectedExitCondition, observing: observedValues, expression: expression, comments: comments(),