From d467a8036cd4be742909ffc058cc3b8d9556f7f4 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Thu, 21 Aug 2025 17:42:06 -0400 Subject: [PATCH 01/24] Add experimental SPI to cancel a running test. This PR introduces `Test.cancel()` and `Test.Case.cancel()` which cancel the current test/suite and the current test case, respectively. For example: ```swift @Test(arguments: [Food.burger, .fries, .iceCream]) func `Food truck is well-stocked`(_ food: Food) throws { if food == .iceCream && Season.current == .winter { try Test.Case.cancel("It's too cold for ice cream.") } // ... } ``` These functions work by cancelling the child task associated with the current test or test case, then throwing an error to end local execution early. Compare `XCTSkip()` which, in Swift, is just a thrown error that the XCTest harness special-cases, or `XCTSkip()` in Objective-C which actually throws an exception to force the caller to exit early. --- Package.swift | 1 + .../Testing/ABI/ABI.Record+Streaming.swift | 8 +- Sources/Testing/CMakeLists.txt | 1 + Sources/Testing/Events/Event.swift | 26 ++- .../Event.HumanReadableOutputRecorder.swift | 75 ++++--- Sources/Testing/Issues/Issue+Recording.swift | 6 + .../Testing/Parameterization/Test.Case.swift | 6 + Sources/Testing/Running/Runner.Plan.swift | 44 +++- .../Testing/Running/Runner.RuntimeState.swift | 8 +- Sources/Testing/Test+Cancellation.swift | 212 ++++++++++++++++++ Sources/Testing/Test.swift | 3 + .../TestingTests/TestCancellationTests.swift | 94 ++++++++ .../TestSupport/TestingAdditions.swift | 14 +- 13 files changed, 454 insertions(+), 44 deletions(-) create mode 100644 Sources/Testing/Test+Cancellation.swift create mode 100644 Tests/TestingTests/TestCancellationTests.swift diff --git a/Package.swift b/Package.swift index 80db6076c..4c58c0a24 100644 --- a/Package.swift +++ b/Package.swift @@ -400,6 +400,7 @@ extension Array where Element == PackageDescription.SwiftSetting { .enableExperimentalFeature("AvailabilityMacro=_regexAPI:macOS 13.0, iOS 16.0, tvOS 16.0, watchOS 9.0"), .enableExperimentalFeature("AvailabilityMacro=_swiftVersionAPI:macOS 13.0, iOS 16.0, tvOS 16.0, watchOS 9.0"), .enableExperimentalFeature("AvailabilityMacro=_typedThrowsAPI:macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0"), + .enableExperimentalFeature("AvailabilityMacro=_asyncUnsafeCurrentTaskAPI:macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0"), .enableExperimentalFeature("AvailabilityMacro=_distantFuture:macOS 99.0, iOS 99.0, watchOS 99.0, tvOS 99.0, visionOS 99.0"), ] diff --git a/Sources/Testing/ABI/ABI.Record+Streaming.swift b/Sources/Testing/ABI/ABI.Record+Streaming.swift index 1aa1362ec..a2afb0b1e 100644 --- a/Sources/Testing/ABI/ABI.Record+Streaming.swift +++ b/Sources/Testing/ABI/ABI.Record+Streaming.swift @@ -47,11 +47,17 @@ extension ABI.Xcode16 { forwardingTo eventHandler: @escaping @Sendable (_ recordJSON: UnsafeRawBufferPointer) -> Void ) -> Event.Handler { return { event, context in - if case .testDiscovered = event.kind { + switch event.kind { + case .testDiscovered: // Discard events of this kind rather than forwarding them to avoid a // crash in Xcode 16 (which does not expect any events to occur before // .runStarted.) return + case .testCancelled, .testCaseCancelled: + // Discard these events as Xcode 16 does not know how to handle them. + return + default: + break } struct EventAndContextSnapshot: Codable { diff --git a/Sources/Testing/CMakeLists.txt b/Sources/Testing/CMakeLists.txt index 531d27cfc..d2cd21ab9 100644 --- a/Sources/Testing/CMakeLists.txt +++ b/Sources/Testing/CMakeLists.txt @@ -93,6 +93,7 @@ add_library(Testing Test.ID.Selection.swift Test.ID.swift Test.swift + Test+Cancellation.swift Test+Discovery.swift Test+Discovery+Legacy.swift Test+Macro.swift diff --git a/Sources/Testing/Events/Event.swift b/Sources/Testing/Events/Event.swift index 0be14ae88..00f81fe5b 100644 --- a/Sources/Testing/Events/Event.swift +++ b/Sources/Testing/Events/Event.swift @@ -74,6 +74,18 @@ public struct Event: Sendable { /// that was passed to the event handler along with this event. case testCaseEnded + /// A test case was cancelled. + /// + /// - Parameters: + /// - skipInfo: A ``SkipInfo`` with details about the cancelled test case. + /// + /// This event is generated by a call to ``Test/Case/cancel(_:sourceLocation:)``. + /// + /// The test case that was cancelled is contained in the ``Event/Context`` + /// instance that was passed to the event handler along with this event. + @_spi(Experimental) + indirect case testCaseCancelled(_ skipInfo: SkipInfo) + /// An expectation was checked with `#expect()` or `#require()`. /// /// - Parameters: @@ -121,6 +133,18 @@ public struct Event: Sendable { /// available from this event's ``Event/testID`` property. indirect case testSkipped(_ skipInfo: SkipInfo) + /// A test was cancelled. + /// + /// - Parameters: + /// - skipInfo: A ``SkipInfo`` with details about the cancelled test. + /// + /// This event is generated by a call to ``Test/cancel(_:sourceLocation:)``. + /// + /// The test that was cancelled is contained in the ``Event/Context`` + /// instance that was passed to the event handler along with this event. + @_spi(Experimental) + indirect case testCancelled(_ skipInfo: SkipInfo) + /// A step in the runner plan ended. /// /// - Parameters: @@ -488,7 +512,7 @@ extension Event.Kind { self = .valueAttached case .testEnded: self = .testEnded - case let .testSkipped(skipInfo): + case let .testCancelled(skipInfo), let .testSkipped(skipInfo), let .testCaseCancelled(skipInfo): self = .testSkipped(skipInfo) case .planStepEnded: self = .planStepEnded diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index 434487e27..7497ad4cd 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -73,6 +73,9 @@ extension Event { /// The number of known issues recorded for the test. var knownIssueCount = 0 + + /// Information about the cancellation of this test or test case. + var cancellationInfo: SkipInfo? } /// Data tracked on a per-test basis. @@ -251,6 +254,7 @@ extension Event.HumanReadableOutputRecorder { 0 } let test = eventContext.test + let testCase = eventContext.testCase let keyPath = eventContext.keyPath let testName = if let test { if let displayName = test.displayName { @@ -310,6 +314,9 @@ extension Event.HumanReadableOutputRecorder { case .testCaseStarted: context.testData[keyPath] = .init(startInstant: instant) + case let .testCancelled(skipInfo), let .testCaseCancelled(skipInfo): + context.testData[keyPath]?.cancellationInfo = skipInfo + default: // These events do not manipulate the context structure. break @@ -404,21 +411,29 @@ extension Event.HumanReadableOutputRecorder { } else { "" } - return if issues.errorIssueCount > 0 { - CollectionOfOne( - Message( - symbol: .fail, - stringValue: "\(_capitalizedTitle(for: test)) \(testName)\(testCasesCount) failed after \(duration)\(issues.description)." - ) - ) + _formattedComments(for: test) + var cancellationComment = "." + let (symbol, verbed): (Event.Symbol, String) + if issues.errorIssueCount > 0 { + (symbol, verbed) = (.fail, "failed") + } else if !test.isParameterized, let cancellationInfo = testData.cancellationInfo { + if let comment = cancellationInfo.comment { + cancellationComment = ": \"\(comment.rawValue)\"" + } + (symbol, verbed) = (.skip, "was cancelled") } else { - [ - Message( - symbol: .pass(knownIssueCount: issues.knownIssueCount), - stringValue: "\(_capitalizedTitle(for: test)) \(testName)\(testCasesCount) passed after \(duration)\(issues.description)." - ) - ] + (symbol, verbed) = (.pass(knownIssueCount: issues.knownIssueCount), "passed") + } + + var result = [ + Message( + symbol: symbol, + stringValue: "\(_capitalizedTitle(for: test)) \(testName)\(testCasesCount) \(verbed) after \(duration)\(issues.description)\(cancellationComment)" + ) + ] + if issues.errorIssueCount > 0 { + result += _formattedComments(for: test) } + return result case let .testSkipped(skipInfo): let test = test! @@ -443,7 +458,7 @@ extension Event.HumanReadableOutputRecorder { } else { 0 } - let labeledArguments = if let testCase = eventContext.testCase { + let labeledArguments = if let testCase { testCase.labeledArguments() } else { "" @@ -523,7 +538,7 @@ extension Event.HumanReadableOutputRecorder { return result case .testCaseStarted: - guard let testCase = eventContext.testCase, testCase.isParameterized, let arguments = testCase.arguments else { + guard let testCase, testCase.isParameterized, let arguments = testCase.arguments else { break } @@ -535,7 +550,7 @@ extension Event.HumanReadableOutputRecorder { ] case .testCaseEnded: - guard verbosity > 0, let testCase = eventContext.testCase, testCase.isParameterized, let arguments = testCase.arguments else { + guard verbosity > 0, let test, let testCase, testCase.isParameterized, let arguments = testCase.arguments else { break } @@ -544,18 +559,28 @@ extension Event.HumanReadableOutputRecorder { let issues = _issueCounts(in: testDataGraph) let duration = testData.startInstant.descriptionOfDuration(to: instant) - let message = if issues.errorIssueCount > 0 { - Message( - symbol: .fail, - stringValue: "Test case passing \(arguments.count.counting("argument")) \(testCase.labeledArguments(includingQualifiedTypeNames: verbosity > 0)) to \(testName) failed after \(duration)\(issues.description)." - ) + var cancellationComment = "." + let (symbol, verbed): (Event.Symbol, String) + if issues.errorIssueCount > 0 { + (symbol, verbed) = (.fail, "failed") + } else if !test.isParameterized, let cancellationInfo = testData.cancellationInfo { + if let comment = cancellationInfo.comment { + cancellationComment = ": \"\(comment.rawValue)\"" + } + (symbol, verbed) = (.skip, "was cancelled") } else { + (symbol, verbed) = (.pass(knownIssueCount: issues.knownIssueCount), "passed") + } + return [ Message( - symbol: .pass(knownIssueCount: issues.knownIssueCount), - stringValue: "Test case passing \(arguments.count.counting("argument")) \(testCase.labeledArguments(includingQualifiedTypeNames: verbosity > 0)) to \(testName) passed after \(duration)\(issues.description)." + symbol: symbol, + stringValue: "Test case passing \(arguments.count.counting("argument")) \(testCase.labeledArguments(includingQualifiedTypeNames: verbosity > 0)) to \(testName) \(verbed) after \(duration)\(issues.description)\(cancellationComment)" ) - } - return [message] + ] + + case .testCancelled, .testCaseCancelled: + // Handled in .testEnded and .testCaseEnded + break case let .iterationEnded(index): guard let iterationStartInstant = context.iterationStartInstant else { diff --git a/Sources/Testing/Issues/Issue+Recording.swift b/Sources/Testing/Issues/Issue+Recording.swift index d99323777..038107dd3 100644 --- a/Sources/Testing/Issues/Issue+Recording.swift +++ b/Sources/Testing/Issues/Issue+Recording.swift @@ -185,6 +185,9 @@ extension Issue { // This error is thrown by expectation checking functions to indicate a // condition evaluated to `false`. Those functions record their own issue, // so we don't need to record another one redundantly. + } catch is SkipInfo { + // This error represents control flow rather than an issue, so we suppress + // it here. } catch { let issue = Issue(for: error, sourceLocation: sourceLocation) issue.record(configuration: configuration) @@ -226,6 +229,9 @@ extension Issue { // This error is thrown by expectation checking functions to indicate a // condition evaluated to `false`. Those functions record their own issue, // so we don't need to record another one redundantly. + } catch is SkipInfo { + // This error represents control flow rather than an issue, so we suppress + // it here. } catch { let issue = Issue(for: error, sourceLocation: sourceLocation) issue.record(configuration: configuration) diff --git a/Sources/Testing/Parameterization/Test.Case.swift b/Sources/Testing/Parameterization/Test.Case.swift index ab9183cf8..375b2f68f 100644 --- a/Sources/Testing/Parameterization/Test.Case.swift +++ b/Sources/Testing/Parameterization/Test.Case.swift @@ -233,6 +233,9 @@ extension Test { /// Do not invoke this closure directly. Always use a ``Runner`` to invoke a /// test or test case. var body: @Sendable () async throws -> Void + + /// The task associated with this instance, if any, guarded by a lock. + nonisolated(unsafe) var unsafeCurrentTask = Locked() } /// A type representing a single parameter to a parameterized test function. @@ -285,6 +288,9 @@ extension Test.Case.Argument.ID: Codable {} extension Test.Parameter: Hashable {} extension Test.Case.Argument.ID: Hashable {} +// MARK: - Cancellation + + #if !SWT_NO_SNAPSHOT_TYPES // MARK: - Snapshotting diff --git a/Sources/Testing/Running/Runner.Plan.swift b/Sources/Testing/Running/Runner.Plan.swift index c89fdecb5..e581bcda8 100644 --- a/Sources/Testing/Running/Runner.Plan.swift +++ b/Sources/Testing/Running/Runner.Plan.swift @@ -193,6 +193,36 @@ extension Runner.Plan { synthesizeSuites(in: &graph, sourceLocation: &sourceLocation) } + /// The basic "run" action. + private static let _runAction = Action.run(options: .init()) + + private static func _determineAction(for test: Test) async -> (Action, (any Error)?) { + await withTaskGroup(returning: (Action, (any Error)?).self) { taskGroup in + taskGroup.addTask { + var action = _runAction + var firstCaughtError: (any Error)? + + await Test.withCurrent(test) { + for trait in test.traits { + do { + try await trait.prepare(for: test) + } catch let error as SkipInfo { + action = .skip(error) + break + } catch { + // Only preserve the first caught error + firstCaughtError = firstCaughtError ?? error + } + } + } + + return (action, firstCaughtError) + } + + return await taskGroup.first { _ in true }! + } + } + /// Construct a graph of runner plan steps for the specified tests. /// /// - Parameters: @@ -211,7 +241,7 @@ extension Runner.Plan { // Convert the list of test into a graph of steps. The actions for these // steps will all be .run() *unless* an error was thrown while examining // them, in which case it will be .recordIssue(). - let runAction = Action.run(options: .init()) + let runAction = _runAction var testGraph = Graph() var actionGraph = Graph(value: runAction) for test in tests { @@ -269,17 +299,7 @@ extension Runner.Plan { // But if any throw another kind of error, keep track of the first error // but continue walking, because if any subsequent traits throw a // `SkipInfo`, the error should not be recorded. - for trait in test.traits { - do { - try await trait.prepare(for: test) - } catch let error as SkipInfo { - action = .skip(error) - break - } catch { - // Only preserve the first caught error - firstCaughtError = firstCaughtError ?? error - } - } + (action, firstCaughtError) = await _determineAction(for: test) // If no trait specified that the test should be skipped, but one did // throw an error, then the action is to record an issue for that error. diff --git a/Sources/Testing/Running/Runner.RuntimeState.swift b/Sources/Testing/Running/Runner.RuntimeState.swift index 9ae299412..9723372eb 100644 --- a/Sources/Testing/Running/Runner.RuntimeState.swift +++ b/Sources/Testing/Running/Runner.RuntimeState.swift @@ -206,7 +206,9 @@ extension Test { static func withCurrent(_ test: Self, perform body: () async throws -> R) async rethrows -> R { var runtimeState = Runner.RuntimeState.current ?? .init() runtimeState.test = test - return try await Runner.RuntimeState.$current.withValue(runtimeState, operation: body) + return try await Runner.RuntimeState.$current.withValue(runtimeState) { + try await test.withUnsafeCurrentTask(body) + } } } @@ -239,7 +241,9 @@ extension Test.Case { static func withCurrent(_ testCase: Self, perform body: () async throws -> R) async rethrows -> R { var runtimeState = Runner.RuntimeState.current ?? .init() runtimeState.testCase = testCase - return try await Runner.RuntimeState.$current.withValue(runtimeState, operation: body) + return try await Runner.RuntimeState.$current.withValue(runtimeState) { + try await testCase.withUnsafeCurrentTask(body) + } } } diff --git a/Sources/Testing/Test+Cancellation.swift b/Sources/Testing/Test+Cancellation.swift new file mode 100644 index 000000000..880e0a9e3 --- /dev/null +++ b/Sources/Testing/Test+Cancellation.swift @@ -0,0 +1,212 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +/// A protocol describing cancellable tests and test cases. +/// +/// This protocol is used to abstract away the common implementation of test and +/// test case cancellation. +protocol TestCancellable: Sendable { + /// The task associated with this instance, if any, guarded by a lock. + var unsafeCurrentTask: Locked { get set } + + /// Make an instance of ``Event/Kind`` appropriate for `self`. + /// + /// - Parameters: + /// - skipInfo: The ``SkipInfo`` structure describing the cancellation. + /// + /// - Returns: An instance of ``Event/Kind`` that describes the cancellation. + func makeCancelledEventKind(with skipInfo: SkipInfo) -> Event.Kind +} + +extension TestCancellable { + /// Call a function while the ``unsafeCurrentTask`` property of this instance + /// is set to the current task. + /// + /// - Parameters: + /// - body: The function to invoke. + /// + /// - Returns: Whatever is returned by `body`. + /// + /// - Throws: Whatever is thrown by `body`. + /// + /// This function sets the ``unsafeCurrentTask`` property, calls `body`, then + /// sets ``unsafeCurrentTask`` back to its previous value. + func withUnsafeCurrentTask(_ body: () async throws -> R) async rethrows -> R { + if #available(_asyncUnsafeCurrentTaskAPI, *) { + return try await _Concurrency.withUnsafeCurrentTask { task in + let oldTask = task + unsafeCurrentTask.withLock { $0 = task } + defer { + unsafeCurrentTask.withLock { $0 = oldTask } + } + return try await body() + } + } else { + return try await body() + } + } +} + +/// The common implementation of cancellation for ``Test`` and ``Test/Case``. +/// +/// - Parameters: +/// - cancellableValue: The test or test case to cancel, or `nil` if neither +/// is set and we need fallback handling. +/// - comment: A comment describing why you are cancelling the test/case. +/// - sourceLocation: The source location to which the testing library will +/// attribute the cancellation. +/// +/// - Throws: An instance of ``SkipInfo`` describing the cancellation. +private func _cancel(_ cancellableValue: (some TestCancellable)?, _ comment: Comment?, sourceLocation: SourceLocation) throws -> Never { + let sourceContext = SourceContext(backtrace: .current(), sourceLocation: sourceLocation) + let skipInfo = SkipInfo(comment: comment, sourceContext: sourceContext) + + if let cancellableValue { + // If the current test case is still running, cancel its task and clear its + // task property (which signals that it has been cancelled.) + let wasRunning = cancellableValue.unsafeCurrentTask.withLock { task in + let result = (task != nil) + task?.cancel() + task = nil + return result + } + + // If we just cancelled the current test case's task, post a corresponding + // event with the relevant skip info. + if wasRunning { + Event.post(cancellableValue.makeCancelledEventKind(with: skipInfo)) + } + } else { + // The current task isn't associated with a test case, so just cancel it + // and (try to) record an API misuse issue. + withUnsafeCurrentTask { task in + task?.cancel() + } + + let issue = Issue( + kind: .apiMisused, + comments: ["Attempted to cancel the current test case, but there is no test case associated with the current task."] + Array(comment), + sourceContext: sourceContext + ) + issue.record() + } + + throw skipInfo +} + +// MARK: - Test cancellation + +extension Test: TestCancellable { + /// Cancel the current test. + /// + /// - Parameters: + /// - comment: A comment describing why you are cancelling the test. + /// - sourceLocation: The source location to which the testing library will + /// attribute the cancellation. + /// + /// - Throws: An error indicating that the current test case has been + /// cancelled. + /// + /// The testing library runs each test in its own task. When you call this + /// function, the testing library cancels the task associated with the current + /// test: + /// + /// ```swift + /// @Test func `Food truck is well-stocked`() throws { + /// guard businessHours.contains(.now) else { + /// try Test.cancel("We're off the clock.") + /// } + /// // ... + /// } + /// ``` + /// + /// If the current test is parameterized, all of its test cases are + /// cancelled. If the current test is a suite, all of its tests are cancelled. + /// If the current test has already been cancelled, this function throws an + /// error but does not attempt to cancel the test a second time. + /// + /// To cancel the current test case but leave other test cases of the current + /// test alone, call ``Test/Case/cancel(_:sourceLocation:)`` instead. + /// + /// - Important: If the current task is not associated with a test (for + /// example, because it was created with [`Task.detached(name:priority:operation:)`](https://developer.apple.com/documentation/swift/task/detached(name:priority:operation:)-795w1)) + /// this function records an issue and cancels the current task. + @_spi(Experimental) + @available(_asyncUnsafeCurrentTaskAPI, *) + public static func cancel( + _ comment: Comment? = nil, + sourceLocation: SourceLocation = #_sourceLocation + ) throws -> Never { + try _cancel(Test.current, comment, sourceLocation: sourceLocation) + } + + func makeCancelledEventKind(with skipInfo: SkipInfo) -> Event.Kind { + .testCancelled(skipInfo) + } +} + +// MARK: - Test case cancellation + +extension Test.Case: TestCancellable { + /// Cancel the current test case. + /// + /// - Parameters: + /// - comment: A comment describing why you are cancelling the test case. + /// - sourceLocation: The source location to which the testing library will + /// attribute the cancellation. + /// + /// - Throws: An error indicating that the current test case has been + /// cancelled. + /// + /// The testing library runs each test case of a test in its own task. When + /// you call this function, the testing library cancels the task associated + /// with the current test case: + /// + /// ```swift + /// @Test(arguments: [Food.burger, .fries, .iceCream]) + /// func `Food truck is well-stocked`(_ food: Food) throws { + /// if food == .iceCream && Season.current == .winter { + /// try Test.Case.cancel("It's too cold for ice cream.") + /// } + /// // ... + /// } + /// ``` + /// + /// If the current test is parameterized, the test's other test cases continue + /// running. If the current test case has already been cancelled, this + /// function throws an error but does not attempt to cancel the test case a + /// second time. + /// + /// To cancel all test cases in the current test, call + /// ``Test/cancel(_:sourceLocation:)`` instead. + /// + /// - Important: If the current task is not associated with a test case (for + /// example, because it was created with [`Task.detached(name:priority:operation:)`](https://developer.apple.com/documentation/swift/task/detached(name:priority:operation:)-795w1)) + /// this function records an issue and cancels the current task. + @_spi(Experimental) + @available(_asyncUnsafeCurrentTaskAPI, *) + public static func cancel( + _ comment: Comment? = nil, + sourceLocation: SourceLocation = #_sourceLocation + ) throws -> Never { + if let test = Test.current, !test.isParameterized { + // The current test is not parameterized, so cancel the whole test rather + // than just the test case. + try _cancel(test, comment, sourceLocation: sourceLocation) + } + + // Cancel the current test case (if it's nil, that's the API misuse path.) + try _cancel(Test.Case.current, comment, sourceLocation: sourceLocation) + } + + func makeCancelledEventKind(with skipInfo: SkipInfo) -> Event.Kind { + .testCaseCancelled(skipInfo) + } +} diff --git a/Sources/Testing/Test.swift b/Sources/Testing/Test.swift index 5f2ac2406..52670ffba 100644 --- a/Sources/Testing/Test.swift +++ b/Sources/Testing/Test.swift @@ -201,6 +201,9 @@ public struct Test: Sendable { @_spi(ForToolsIntegrationOnly) public var isSynthesized: Bool = false + /// The task associated with this test, if any, guarded by a lock. + nonisolated(unsafe) var unsafeCurrentTask = Locked() + /// Initialize an instance of this type representing a test suite type. init( displayName: String? = nil, diff --git a/Tests/TestingTests/TestCancellationTests.swift b/Tests/TestingTests/TestCancellationTests.swift new file mode 100644 index 000000000..b626493c3 --- /dev/null +++ b/Tests/TestingTests/TestCancellationTests.swift @@ -0,0 +1,94 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +@testable @_spi(Experimental) @_spi(ForToolsIntegrationOnly) import Testing + +struct `Test cancellation tests` { + func testCancellation(testCancelled: Int = 0, testSkipped: Int = 0, testCaseCancelled: Int = 0, issueRecorded: Int = 0, _ body: @Sendable (Configuration) async -> Void) async { + await confirmation("Test cancelled", expectedCount: testCancelled) { testCancelled in + await confirmation("Test skipped", expectedCount: testSkipped) { testSkipped in + await confirmation("Test case cancelled", expectedCount: testCaseCancelled) { testCaseCancelled in + await confirmation("Issue recorded", expectedCount: issueRecorded) { issueRecorded in + var configuration = Configuration() + configuration.eventHandler = { event, _ in + switch event.kind { + case .testCancelled: + testCancelled() + case .testSkipped: + testSkipped() + case .testCaseCancelled: + testCaseCancelled() + case .issueRecorded: + issueRecorded() + default: + break + } + } + await body(configuration) + } + } + } + } + } + + @Test func `Cancelling a test`() async { + await testCancellation(testCancelled: 1) { configuration in + await Test { + try Test.cancel("Cancelled test") + }.run(configuration: configuration) + } + } + + @Test func `Cancelling a non-parameterized test via Test.Case.cancel()`() async { + await testCancellation(testCancelled: 1) { configuration in + await Test { + try Test.Case.cancel("Cancelled test") + }.run(configuration: configuration) + } + } + + @Test func `Cancelling a test case in a parameterized test`() async { + await testCancellation(testCaseCancelled: 5, issueRecorded: 5) { configuration in + await Test(arguments: 0 ..< 10) { i in + if (i % 2) == 0 { + try Test.Case.cancel("\(i) is even!") + } + Issue.record("\(i) records an issue!") + }.run(configuration: configuration) + } + } + + @Test func `Cancelling an entire parameterized test`() async { + await testCancellation(testCancelled: 1) { configuration in + // .serialized to ensure that none of the cases complete before the first + // one cancels the test. + await Test(.serialized, arguments: 0 ..< 10) { i in + if i == 0 { + try Test.cancel("\(i) cancelled the test") + } + Issue.record("\(i) records an issue!") + }.run(configuration: configuration) + } + } + + struct CancelledTrait: TestTrait { + func prepare(for test: Test) async throws { + try Test.cancel("Cancelled from trait") + } + } + + @Test func `Cancelling a test case while evaluating traits skips the test`() async { + await testCancellation(testSkipped: 1) { configuration in + await Test(CancelledTrait()) { + Issue.record("Recorded an issue!") + }.run(configuration: configuration) + } + } +} diff --git a/Tests/TestingTests/TestSupport/TestingAdditions.swift b/Tests/TestingTests/TestSupport/TestingAdditions.swift index 5c596785e..30d53ce7f 100644 --- a/Tests/TestingTests/TestSupport/TestingAdditions.swift +++ b/Tests/TestingTests/TestSupport/TestingAdditions.swift @@ -187,7 +187,9 @@ extension Test { init( _ traits: any TestTrait..., arguments collection: C, - parameters: [Parameter] = [], + parameters: [Parameter] = [ + Parameter(index: 0, firstName: "x", type: C.Element.self), + ], sourceLocation: SourceLocation = #_sourceLocation, column: Int = #column, name: String = #function, @@ -216,7 +218,10 @@ extension Test { init( _ traits: any TestTrait..., arguments collection1: C1, _ collection2: C2, - parameters: [Parameter] = [], + parameters: [Parameter] = [ + Parameter(index: 0, firstName: "x", type: C1.Element.self), + Parameter(index: 1, firstName: "y", type: C2.Element.self), + ], sourceLocation: SourceLocation = #_sourceLocation, name: String = #function, testFunction: @escaping @Sendable (C1.Element, C2.Element) async throws -> Void @@ -239,7 +244,10 @@ extension Test { init( _ traits: any TestTrait..., arguments zippedCollections: Zip2Sequence, - parameters: [Parameter] = [], + parameters: [Parameter] = [ + Parameter(index: 0, firstName: "x", type: C1.Element.self), + Parameter(index: 1, firstName: "y", type: C2.Element.self), + ], sourceLocation: SourceLocation = #_sourceLocation, name: String = #function, testFunction: @escaping @Sendable ((C1.Element, C2.Element)) async throws -> Void From 4dce6f8152a58005761354ab725585d52cd6f90c Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Thu, 21 Aug 2025 18:04:57 -0400 Subject: [PATCH 02/24] Availability --- Tests/TestingTests/TestCancellationTests.swift | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Tests/TestingTests/TestCancellationTests.swift b/Tests/TestingTests/TestCancellationTests.swift index b626493c3..dfda8e5c3 100644 --- a/Tests/TestingTests/TestCancellationTests.swift +++ b/Tests/TestingTests/TestCancellationTests.swift @@ -38,6 +38,7 @@ struct `Test cancellation tests` { } } + @available(_asyncUnsafeCurrentTaskAPI, *) @Test func `Cancelling a test`() async { await testCancellation(testCancelled: 1) { configuration in await Test { @@ -46,6 +47,7 @@ struct `Test cancellation tests` { } } + @available(_asyncUnsafeCurrentTaskAPI, *) @Test func `Cancelling a non-parameterized test via Test.Case.cancel()`() async { await testCancellation(testCancelled: 1) { configuration in await Test { @@ -54,6 +56,7 @@ struct `Test cancellation tests` { } } + @available(_asyncUnsafeCurrentTaskAPI, *) @Test func `Cancelling a test case in a parameterized test`() async { await testCancellation(testCaseCancelled: 5, issueRecorded: 5) { configuration in await Test(arguments: 0 ..< 10) { i in @@ -65,6 +68,7 @@ struct `Test cancellation tests` { } } + @available(_asyncUnsafeCurrentTaskAPI, *) @Test func `Cancelling an entire parameterized test`() async { await testCancellation(testCancelled: 1) { configuration in // .serialized to ensure that none of the cases complete before the first @@ -78,12 +82,14 @@ struct `Test cancellation tests` { } } + @available(_asyncUnsafeCurrentTaskAPI, *) struct CancelledTrait: TestTrait { func prepare(for test: Test) async throws { try Test.cancel("Cancelled from trait") } } + @available(_asyncUnsafeCurrentTaskAPI, *) @Test func `Cancelling a test case while evaluating traits skips the test`() async { await testCancellation(testSkipped: 1) { configuration in await Test(CancelledTrait()) { From 367c707ab895e02e07793697d0bdd8c58497cf22 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Thu, 21 Aug 2025 18:16:58 -0400 Subject: [PATCH 03/24] Lower availability with a fallback implementation --- Sources/Testing/Test+Cancellation.swift | 9 +++++++-- Tests/TestingTests/TestCancellationTests.swift | 6 ------ cmake/modules/shared/AvailabilityDefinitions.cmake | 1 + 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Sources/Testing/Test+Cancellation.swift b/Sources/Testing/Test+Cancellation.swift index 880e0a9e3..86c390b2d 100644 --- a/Sources/Testing/Test+Cancellation.swift +++ b/Sources/Testing/Test+Cancellation.swift @@ -49,6 +49,13 @@ extension TestCancellable { return try await body() } } else { + let oldTask = _Concurrency.withUnsafeCurrentTask { task in + unsafeCurrentTask.withLock { $0 = task } + return task + } + defer { + unsafeCurrentTask.withLock { $0 = oldTask } + } return try await body() } } @@ -139,7 +146,6 @@ extension Test: TestCancellable { /// example, because it was created with [`Task.detached(name:priority:operation:)`](https://developer.apple.com/documentation/swift/task/detached(name:priority:operation:)-795w1)) /// this function records an issue and cancels the current task. @_spi(Experimental) - @available(_asyncUnsafeCurrentTaskAPI, *) public static func cancel( _ comment: Comment? = nil, sourceLocation: SourceLocation = #_sourceLocation @@ -191,7 +197,6 @@ extension Test.Case: TestCancellable { /// example, because it was created with [`Task.detached(name:priority:operation:)`](https://developer.apple.com/documentation/swift/task/detached(name:priority:operation:)-795w1)) /// this function records an issue and cancels the current task. @_spi(Experimental) - @available(_asyncUnsafeCurrentTaskAPI, *) public static func cancel( _ comment: Comment? = nil, sourceLocation: SourceLocation = #_sourceLocation diff --git a/Tests/TestingTests/TestCancellationTests.swift b/Tests/TestingTests/TestCancellationTests.swift index dfda8e5c3..b626493c3 100644 --- a/Tests/TestingTests/TestCancellationTests.swift +++ b/Tests/TestingTests/TestCancellationTests.swift @@ -38,7 +38,6 @@ struct `Test cancellation tests` { } } - @available(_asyncUnsafeCurrentTaskAPI, *) @Test func `Cancelling a test`() async { await testCancellation(testCancelled: 1) { configuration in await Test { @@ -47,7 +46,6 @@ struct `Test cancellation tests` { } } - @available(_asyncUnsafeCurrentTaskAPI, *) @Test func `Cancelling a non-parameterized test via Test.Case.cancel()`() async { await testCancellation(testCancelled: 1) { configuration in await Test { @@ -56,7 +54,6 @@ struct `Test cancellation tests` { } } - @available(_asyncUnsafeCurrentTaskAPI, *) @Test func `Cancelling a test case in a parameterized test`() async { await testCancellation(testCaseCancelled: 5, issueRecorded: 5) { configuration in await Test(arguments: 0 ..< 10) { i in @@ -68,7 +65,6 @@ struct `Test cancellation tests` { } } - @available(_asyncUnsafeCurrentTaskAPI, *) @Test func `Cancelling an entire parameterized test`() async { await testCancellation(testCancelled: 1) { configuration in // .serialized to ensure that none of the cases complete before the first @@ -82,14 +78,12 @@ struct `Test cancellation tests` { } } - @available(_asyncUnsafeCurrentTaskAPI, *) struct CancelledTrait: TestTrait { func prepare(for test: Test) async throws { try Test.cancel("Cancelled from trait") } } - @available(_asyncUnsafeCurrentTaskAPI, *) @Test func `Cancelling a test case while evaluating traits skips the test`() async { await testCancellation(testSkipped: 1) { configuration in await Test(CancelledTrait()) { diff --git a/cmake/modules/shared/AvailabilityDefinitions.cmake b/cmake/modules/shared/AvailabilityDefinitions.cmake index 2124a32be..3bcfd8efe 100644 --- a/cmake/modules/shared/AvailabilityDefinitions.cmake +++ b/cmake/modules/shared/AvailabilityDefinitions.cmake @@ -15,4 +15,5 @@ add_compile_options( "SHELL:$<$:-Xfrontend -define-availability -Xfrontend \"_regexAPI:macOS 13.0, iOS 16.0, tvOS 16.0, watchOS 9.0\">" "SHELL:$<$:-Xfrontend -define-availability -Xfrontend \"_swiftVersionAPI:macOS 13.0, iOS 16.0, tvOS 16.0, watchOS 9.0\">" "SHELL:$<$:-Xfrontend -define-availability -Xfrontend \"_typedThrowsAPI:macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0\">" + "SHELL:$<$:-Xfrontend -define-availability -Xfrontend \"_asyncUnsafeCurrentTaskAPI:macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0\">" "SHELL:$<$:-Xfrontend -define-availability -Xfrontend \"_distantFuture:macOS 99.0, iOS 99.0, watchOS 99.0, tvOS 99.0, visionOS 99.0\">") From 576a478d650be2fd53f828439753d29db1df3bde Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Thu, 21 Aug 2025 18:19:08 -0400 Subject: [PATCH 04/24] Wrong logic to get old task (facepalm) --- Sources/Testing/Test+Cancellation.swift | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/Sources/Testing/Test+Cancellation.swift b/Sources/Testing/Test+Cancellation.swift index 86c390b2d..aaeb99404 100644 --- a/Sources/Testing/Test+Cancellation.swift +++ b/Sources/Testing/Test+Cancellation.swift @@ -41,8 +41,11 @@ extension TestCancellable { func withUnsafeCurrentTask(_ body: () async throws -> R) async rethrows -> R { if #available(_asyncUnsafeCurrentTaskAPI, *) { return try await _Concurrency.withUnsafeCurrentTask { task in - let oldTask = task - unsafeCurrentTask.withLock { $0 = task } + let oldTask = unsafeCurrentTask.withLock { unsafeCurrentTask in + let oldTask = unsafeCurrentTask + unsafeCurrentTask = task + return oldTask + } defer { unsafeCurrentTask.withLock { $0 = oldTask } } @@ -50,8 +53,11 @@ extension TestCancellable { } } else { let oldTask = _Concurrency.withUnsafeCurrentTask { task in - unsafeCurrentTask.withLock { $0 = task } - return task + unsafeCurrentTask.withLock { unsafeCurrentTask in + let oldTask = unsafeCurrentTask + unsafeCurrentTask = task + return oldTask + } } defer { unsafeCurrentTask.withLock { $0 = oldTask } From be6d9bd0e52db83d90780e54452b48f9c82ae800 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Mon, 25 Aug 2025 16:24:26 -0400 Subject: [PATCH 05/24] Record issues that occur during testing --- Tests/TestingTests/TestCancellationTests.swift | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Tests/TestingTests/TestCancellationTests.swift b/Tests/TestingTests/TestCancellationTests.swift index b626493c3..bda91b27e 100644 --- a/Tests/TestingTests/TestCancellationTests.swift +++ b/Tests/TestingTests/TestCancellationTests.swift @@ -15,7 +15,7 @@ struct `Test cancellation tests` { await confirmation("Test cancelled", expectedCount: testCancelled) { testCancelled in await confirmation("Test skipped", expectedCount: testSkipped) { testSkipped in await confirmation("Test case cancelled", expectedCount: testCaseCancelled) { testCaseCancelled in - await confirmation("Issue recorded", expectedCount: issueRecorded) { issueRecorded in + await confirmation("Issue recorded", expectedCount: issueRecorded) { [issueRecordedCount = issueRecorded] issueRecorded in var configuration = Configuration() configuration.eventHandler = { event, _ in switch event.kind { @@ -25,7 +25,10 @@ struct `Test cancellation tests` { testSkipped() case .testCaseCancelled: testCaseCancelled() - case .issueRecorded: + case let .issueRecorded(issue): + if issueRecordedCount == 0 { + issue.record() + } issueRecorded() default: break From 5cc0093f2d05fde155853535724bd553f960773c Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Mon, 25 Aug 2025 16:51:49 -0400 Subject: [PATCH 06/24] Catch CancellationError from within a test and treat it as cancelling the test (if the test's task is actually cancelled) --- Sources/Testing/Issues/Issue+Recording.swift | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/Sources/Testing/Issues/Issue+Recording.swift b/Sources/Testing/Issues/Issue+Recording.swift index 038107dd3..67a44a056 100644 --- a/Sources/Testing/Issues/Issue+Recording.swift +++ b/Sources/Testing/Issues/Issue+Recording.swift @@ -188,6 +188,15 @@ extension Issue { } catch is SkipInfo { // This error represents control flow rather than an issue, so we suppress // it here. + } catch is CancellationError where Task.isCancelled { + // This error also represents control flow. It should cause the current + // test case to be cancelled, or the current test if there is no current + // test case. + if Test.Case.current != nil { + _ = try? Test.Case.cancel(sourceLocation: sourceLocation) + } else { + _ = try? Test.cancel(sourceLocation: sourceLocation) + } } catch { let issue = Issue(for: error, sourceLocation: sourceLocation) issue.record(configuration: configuration) @@ -232,6 +241,15 @@ extension Issue { } catch is SkipInfo { // This error represents control flow rather than an issue, so we suppress // it here. + } catch is CancellationError where Task.isCancelled { + // This error also represents control flow. It should cause the current + // test case to be cancelled, or the current test if there is no current + // test case. + if Test.Case.current != nil { + _ = try? Test.Case.cancel(sourceLocation: sourceLocation) + } else { + _ = try? Test.cancel(sourceLocation: sourceLocation) + } } catch { let issue = Issue(for: error, sourceLocation: sourceLocation) issue.record(configuration: configuration) From 239088564a7a97c39a9d18dacbf2de154e8a0b83 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Mon, 25 Aug 2025 18:12:18 -0400 Subject: [PATCH 07/24] Interop with task cancellation and CancellationError --- Package.swift | 1 - Sources/Testing/Issues/Issue+Recording.swift | 24 +--- .../Testing/Running/Runner.RuntimeState.swift | 4 +- Sources/Testing/Running/Runner.swift | 14 +-- Sources/Testing/Test+Cancellation.swift | 110 ++++++++++-------- .../TestingTests/TestCancellationTests.swift | 27 ++++- .../shared/AvailabilityDefinitions.cmake | 1 - 7 files changed, 99 insertions(+), 82 deletions(-) diff --git a/Package.swift b/Package.swift index 4c58c0a24..80db6076c 100644 --- a/Package.swift +++ b/Package.swift @@ -400,7 +400,6 @@ extension Array where Element == PackageDescription.SwiftSetting { .enableExperimentalFeature("AvailabilityMacro=_regexAPI:macOS 13.0, iOS 16.0, tvOS 16.0, watchOS 9.0"), .enableExperimentalFeature("AvailabilityMacro=_swiftVersionAPI:macOS 13.0, iOS 16.0, tvOS 16.0, watchOS 9.0"), .enableExperimentalFeature("AvailabilityMacro=_typedThrowsAPI:macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0"), - .enableExperimentalFeature("AvailabilityMacro=_asyncUnsafeCurrentTaskAPI:macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0"), .enableExperimentalFeature("AvailabilityMacro=_distantFuture:macOS 99.0, iOS 99.0, watchOS 99.0, tvOS 99.0, visionOS 99.0"), ] diff --git a/Sources/Testing/Issues/Issue+Recording.swift b/Sources/Testing/Issues/Issue+Recording.swift index 67a44a056..a53d3e5c0 100644 --- a/Sources/Testing/Issues/Issue+Recording.swift +++ b/Sources/Testing/Issues/Issue+Recording.swift @@ -185,18 +185,10 @@ extension Issue { // This error is thrown by expectation checking functions to indicate a // condition evaluated to `false`. Those functions record their own issue, // so we don't need to record another one redundantly. - } catch is SkipInfo { + } catch is SkipInfo, + is CancellationError where Task.isCancelled { // This error represents control flow rather than an issue, so we suppress // it here. - } catch is CancellationError where Task.isCancelled { - // This error also represents control flow. It should cause the current - // test case to be cancelled, or the current test if there is no current - // test case. - if Test.Case.current != nil { - _ = try? Test.Case.cancel(sourceLocation: sourceLocation) - } else { - _ = try? Test.cancel(sourceLocation: sourceLocation) - } } catch { let issue = Issue(for: error, sourceLocation: sourceLocation) issue.record(configuration: configuration) @@ -238,18 +230,10 @@ extension Issue { // This error is thrown by expectation checking functions to indicate a // condition evaluated to `false`. Those functions record their own issue, // so we don't need to record another one redundantly. - } catch is SkipInfo { + } catch is SkipInfo, + is CancellationError where Task.isCancelled { // This error represents control flow rather than an issue, so we suppress // it here. - } catch is CancellationError where Task.isCancelled { - // This error also represents control flow. It should cause the current - // test case to be cancelled, or the current test if there is no current - // test case. - if Test.Case.current != nil { - _ = try? Test.Case.cancel(sourceLocation: sourceLocation) - } else { - _ = try? Test.cancel(sourceLocation: sourceLocation) - } } catch { let issue = Issue(for: error, sourceLocation: sourceLocation) issue.record(configuration: configuration) diff --git a/Sources/Testing/Running/Runner.RuntimeState.swift b/Sources/Testing/Running/Runner.RuntimeState.swift index 9723372eb..e88cea60b 100644 --- a/Sources/Testing/Running/Runner.RuntimeState.swift +++ b/Sources/Testing/Running/Runner.RuntimeState.swift @@ -207,7 +207,7 @@ extension Test { var runtimeState = Runner.RuntimeState.current ?? .init() runtimeState.test = test return try await Runner.RuntimeState.$current.withValue(runtimeState) { - try await test.withUnsafeCurrentTask(body) + try await test.withCancellationHandling(body) } } } @@ -242,7 +242,7 @@ extension Test.Case { var runtimeState = Runner.RuntimeState.current ?? .init() runtimeState.testCase = testCase return try await Runner.RuntimeState.$current.withValue(runtimeState) { - try await testCase.withUnsafeCurrentTask(body) + try await testCase.withCancellationHandling(body) } } } diff --git a/Sources/Testing/Running/Runner.swift b/Sources/Testing/Running/Runner.swift index bd1167b8e..14de2a15b 100644 --- a/Sources/Testing/Running/Runner.swift +++ b/Sources/Testing/Running/Runner.swift @@ -159,7 +159,7 @@ extension Runner { try await withThrowingTaskGroup { taskGroup in for element in sequence { // Each element gets its own subtask to run in. - _ = taskGroup.addTaskUnlessCancelled { + taskGroup.addTask { try await body(element) } @@ -190,9 +190,6 @@ extension Runner { /// /// - ``Runner/run()`` private static func _runStep(atRootOf stepGraph: Graph) async throws { - // Exit early if the task has already been cancelled. - try Task.checkCancellation() - // Whether to send a `.testEnded` event at the end of running this step. // Some steps' actions may not require a final event to be sent — for // example, a skip event only sends `.testSkipped`. @@ -243,6 +240,9 @@ extension Runner { if let step = stepGraph.value, case .run = step.action { await Test.withCurrent(step.test) { _ = await Issue.withErrorRecording(at: step.test.sourceLocation, configuration: configuration) { + // Exit early if the task has already been cancelled. + try Task.checkCancellation() + try await _applyScopingTraits(for: step.test, testCase: nil) { // Run the test function at this step (if one is present.) if let testCases = step.test.testCases { @@ -355,9 +355,6 @@ extension Runner { /// This function sets ``Test/Case/current``, then invokes the test case's /// body closure. private static func _runTestCase(_ testCase: Test.Case, within step: Plan.Step) async throws { - // Exit early if the task has already been cancelled. - try Task.checkCancellation() - let configuration = _configuration Event.post(.testCaseStarted, for: (step.test, testCase), configuration: configuration) @@ -368,6 +365,9 @@ extension Runner { await Test.Case.withCurrent(testCase) { let sourceLocation = step.test.sourceLocation await Issue.withErrorRecording(at: sourceLocation, configuration: configuration) { + // Exit early if the task has already been cancelled. + try Task.checkCancellation() + try await withTimeLimit(for: step.test, configuration: configuration) { try await _applyScopingTraits(for: step.test, testCase: testCase) { try await testCase.body() diff --git a/Sources/Testing/Test+Cancellation.swift b/Sources/Testing/Test+Cancellation.swift index aaeb99404..332393afb 100644 --- a/Sources/Testing/Test+Cancellation.swift +++ b/Sources/Testing/Test+Cancellation.swift @@ -38,31 +38,29 @@ extension TestCancellable { /// /// This function sets the ``unsafeCurrentTask`` property, calls `body`, then /// sets ``unsafeCurrentTask`` back to its previous value. - func withUnsafeCurrentTask(_ body: () async throws -> R) async rethrows -> R { - if #available(_asyncUnsafeCurrentTaskAPI, *) { - return try await _Concurrency.withUnsafeCurrentTask { task in - let oldTask = unsafeCurrentTask.withLock { unsafeCurrentTask in - let oldTask = unsafeCurrentTask - unsafeCurrentTask = task - return oldTask - } - defer { - unsafeCurrentTask.withLock { $0 = oldTask } - } - return try await body() + func withCancellationHandling(_ body: () async throws -> R) async rethrows -> R { + // TODO: adopt async withUnsafeCurrentTask() when our deployment target allows + let oldTask = withUnsafeCurrentTask { task in + unsafeCurrentTask.withLock { unsafeCurrentTask in + let oldTask = unsafeCurrentTask + unsafeCurrentTask = task + return oldTask } - } else { - let oldTask = _Concurrency.withUnsafeCurrentTask { task in - unsafeCurrentTask.withLock { unsafeCurrentTask in - let oldTask = unsafeCurrentTask - unsafeCurrentTask = task - return oldTask - } - } - defer { - unsafeCurrentTask.withLock { $0 = oldTask } + } + defer { + unsafeCurrentTask.withLock { $0 = oldTask } + } + + return try await withTaskCancellationHandler { + try await body() + } onCancel: { + // The current task was cancelled, so cancel the test case or test + // associated with it. + if Test.Case.current != nil { + _ = try? Test.Case.cancel(comment: nil, sourceLocation: nil) + } else if let test = Test.current { + _ = try? _cancel(test, nil, sourceLocation: nil) } - return try await body() } } } @@ -74,26 +72,26 @@ extension TestCancellable { /// is set and we need fallback handling. /// - comment: A comment describing why you are cancelling the test/case. /// - sourceLocation: The source location to which the testing library will -/// attribute the cancellation. +/// attribute the cancellation, if available. /// /// - Throws: An instance of ``SkipInfo`` describing the cancellation. -private func _cancel(_ cancellableValue: (some TestCancellable)?, _ comment: Comment?, sourceLocation: SourceLocation) throws -> Never { +private func _cancel(_ cancellableValue: (some TestCancellable)?, _ comment: Comment?, sourceLocation: SourceLocation?) throws -> Never { let sourceContext = SourceContext(backtrace: .current(), sourceLocation: sourceLocation) let skipInfo = SkipInfo(comment: comment, sourceContext: sourceContext) if let cancellableValue { // If the current test case is still running, cancel its task and clear its // task property (which signals that it has been cancelled.) - let wasRunning = cancellableValue.unsafeCurrentTask.withLock { task in - let result = (task != nil) - task?.cancel() + let task = cancellableValue.unsafeCurrentTask.withLock { task in + let result = task task = nil return result } + task?.cancel() // If we just cancelled the current test case's task, post a corresponding // event with the relevant skip info. - if wasRunning { + if task != nil { Event.post(cancellableValue.makeCancelledEventKind(with: skipInfo)) } } else { @@ -105,7 +103,7 @@ private func _cancel(_ cancellableValue: (some TestCancellable)?, _ comment: Com let issue = Issue( kind: .apiMisused, - comments: ["Attempted to cancel the current test case, but there is no test case associated with the current task."] + Array(comment), + comments: ["Attempted to cancel the current test or case, but one is not associated with the current task."] + Array(comment), sourceContext: sourceContext ) issue.record() @@ -140,10 +138,11 @@ extension Test: TestCancellable { /// } /// ``` /// - /// If the current test is parameterized, all of its test cases are - /// cancelled. If the current test is a suite, all of its tests are cancelled. - /// If the current test has already been cancelled, this function throws an - /// error but does not attempt to cancel the test a second time. + /// If the current test is parameterized, all of its pending and running test + /// cases are cancelled. If the current test is a suite, all of its pending + /// and running tests are cancelled. If you have already cancelled the current + /// test or if it has already finished running, this function throws an error + /// but does not attempt to cancel the test a second time. /// /// To cancel the current test case but leave other test cases of the current /// test alone, call ``Test/Case/cancel(_:sourceLocation:)`` instead. @@ -152,10 +151,7 @@ extension Test: TestCancellable { /// example, because it was created with [`Task.detached(name:priority:operation:)`](https://developer.apple.com/documentation/swift/task/detached(name:priority:operation:)-795w1)) /// this function records an issue and cancels the current task. @_spi(Experimental) - public static func cancel( - _ comment: Comment? = nil, - sourceLocation: SourceLocation = #_sourceLocation - ) throws -> Never { + public static func cancel(_ comment: Comment? = nil, sourceLocation: SourceLocation = #_sourceLocation) throws -> Never { try _cancel(Test.current, comment, sourceLocation: sourceLocation) } @@ -167,6 +163,30 @@ extension Test: TestCancellable { // MARK: - Test case cancellation extension Test.Case: TestCancellable { + /// The implementation of ``cancel(_:sourceLocation:)``, but able to take a + /// `nil` value as its `sourceLocation` argument. + /// + /// - Parameters: + /// - comment: A comment describing why you are cancelling the test. + /// - sourceLocation: The source location to which the testing library will + /// attribute the cancellation. + /// + /// - Throws: An error indicating that the current test case has been + /// cancelled. + /// + /// This overload of `cancel()` is factored out so we can call it with a `nil` + /// source location in ``withCancellationHandling(_:)``. + fileprivate static func cancel(comment: Comment?, sourceLocation: SourceLocation?) throws -> Never { + if let test = Test.current, !test.isParameterized { + // The current test is not parameterized, so cancel the whole test rather + // than just the test case. + try _cancel(test, comment, sourceLocation: sourceLocation) + } + + // Cancel the current test case (if it's nil, that's the API misuse path.) + try _cancel(Test.Case.current, comment, sourceLocation: sourceLocation) + } + /// Cancel the current test case. /// /// - Parameters: @@ -203,18 +223,8 @@ extension Test.Case: TestCancellable { /// example, because it was created with [`Task.detached(name:priority:operation:)`](https://developer.apple.com/documentation/swift/task/detached(name:priority:operation:)-795w1)) /// this function records an issue and cancels the current task. @_spi(Experimental) - public static func cancel( - _ comment: Comment? = nil, - sourceLocation: SourceLocation = #_sourceLocation - ) throws -> Never { - if let test = Test.current, !test.isParameterized { - // The current test is not parameterized, so cancel the whole test rather - // than just the test case. - try _cancel(test, comment, sourceLocation: sourceLocation) - } - - // Cancel the current test case (if it's nil, that's the API misuse path.) - try _cancel(Test.Case.current, comment, sourceLocation: sourceLocation) + public static func cancel(_ comment: Comment? = nil, sourceLocation: SourceLocation = #_sourceLocation) throws -> Never { + try cancel(comment: comment, sourceLocation: sourceLocation) } func makeCancelledEventKind(with skipInfo: SkipInfo) -> Event.Kind { diff --git a/Tests/TestingTests/TestCancellationTests.swift b/Tests/TestingTests/TestCancellationTests.swift index bda91b27e..519eaa85f 100644 --- a/Tests/TestingTests/TestCancellationTests.swift +++ b/Tests/TestingTests/TestCancellationTests.swift @@ -69,7 +69,7 @@ struct `Test cancellation tests` { } @Test func `Cancelling an entire parameterized test`() async { - await testCancellation(testCancelled: 1) { configuration in + await testCancellation(testCancelled: 1, testCaseCancelled: 10) { configuration in // .serialized to ensure that none of the cases complete before the first // one cancels the test. await Test(.serialized, arguments: 0 ..< 10) { i in @@ -81,6 +81,31 @@ struct `Test cancellation tests` { } } + @Test func `Cancelling a test by cancelling its task (throwing)`() async { + await testCancellation(testCancelled: 1) { configuration in + await Test { + withUnsafeCurrentTask { $0?.cancel() } + try Task.checkCancellation() + }.run(configuration: configuration) + } + } + + @Test func `Cancelling a test by cancelling its task (returning)`() async { + await testCancellation(testCancelled: 1) { configuration in + await Test { + withUnsafeCurrentTask { $0?.cancel() } + }.run(configuration: configuration) + } + } + + @Test func `Throwing CancellationError without cancelling the test task`() async { + await testCancellation(issueRecorded: 1) { configuration in + await Test { + throw CancellationError() + }.run(configuration: configuration) + } + } + struct CancelledTrait: TestTrait { func prepare(for test: Test) async throws { try Test.cancel("Cancelled from trait") diff --git a/cmake/modules/shared/AvailabilityDefinitions.cmake b/cmake/modules/shared/AvailabilityDefinitions.cmake index 3bcfd8efe..2124a32be 100644 --- a/cmake/modules/shared/AvailabilityDefinitions.cmake +++ b/cmake/modules/shared/AvailabilityDefinitions.cmake @@ -15,5 +15,4 @@ add_compile_options( "SHELL:$<$:-Xfrontend -define-availability -Xfrontend \"_regexAPI:macOS 13.0, iOS 16.0, tvOS 16.0, watchOS 9.0\">" "SHELL:$<$:-Xfrontend -define-availability -Xfrontend \"_swiftVersionAPI:macOS 13.0, iOS 16.0, tvOS 16.0, watchOS 9.0\">" "SHELL:$<$:-Xfrontend -define-availability -Xfrontend \"_typedThrowsAPI:macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0\">" - "SHELL:$<$:-Xfrontend -define-availability -Xfrontend \"_asyncUnsafeCurrentTaskAPI:macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0\">" "SHELL:$<$:-Xfrontend -define-availability -Xfrontend \"_distantFuture:macOS 99.0, iOS 99.0, watchOS 99.0, tvOS 99.0, visionOS 99.0\">") From d64d0197a4f23425611fcd812ab3b2b698f45a11 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Mon, 25 Aug 2025 20:18:13 -0400 Subject: [PATCH 08/24] Incorporate my own feedback --- .../Testing/ABI/ABI.Record+Streaming.swift | 3 +- .../ABI/Encoded/ABI.EncodedEvent.swift | 19 ++- .../Testing/Parameterization/Test.Case.swift | 6 - Sources/Testing/Running/Runner.Plan.swift | 19 ++- Sources/Testing/Test+Cancellation.swift | 123 +++++++++++------- Sources/Testing/Test.swift | 3 - 6 files changed, 115 insertions(+), 58 deletions(-) diff --git a/Sources/Testing/ABI/ABI.Record+Streaming.swift b/Sources/Testing/ABI/ABI.Record+Streaming.swift index a2afb0b1e..ab22714ae 100644 --- a/Sources/Testing/ABI/ABI.Record+Streaming.swift +++ b/Sources/Testing/ABI/ABI.Record+Streaming.swift @@ -54,7 +54,8 @@ extension ABI.Xcode16 { // .runStarted.) return case .testCancelled, .testCaseCancelled: - // Discard these events as Xcode 16 does not know how to handle them. + // Discard these events as Xcode 16 does not know how to handle them and + // may crash if they arrive during trait evaluation. return default: break diff --git a/Sources/Testing/ABI/Encoded/ABI.EncodedEvent.swift b/Sources/Testing/ABI/Encoded/ABI.EncodedEvent.swift index 73e7db2ac..09bc41a63 100644 --- a/Sources/Testing/ABI/Encoded/ABI.EncodedEvent.swift +++ b/Sources/Testing/ABI/Encoded/ABI.EncodedEvent.swift @@ -29,8 +29,10 @@ extension ABI { case issueRecorded case valueAttached case testCaseEnded + case testCaseCancelled = "_testCaseCancelled" case testEnded case testSkipped + case testCancelled = "_testCancelled" case runEnded } @@ -64,6 +66,12 @@ extension ABI { /// - Warning: Test cases are not yet part of the JSON schema. var _testCase: EncodedTestCase? + /// A source location associated with this event, if any. + /// + /// - Warning: Source locations at this level of the JSON schema are not yet + /// part of said JSON schema. + var _sourceLocation: SourceLocation? + init?(encoding event: borrowing Event, in eventContext: borrowing Event.Context, messages: borrowing [Event.HumanReadableOutputRecorder.Message]) { switch event.kind { case .runStarted: @@ -78,18 +86,27 @@ extension ABI { case let .issueRecorded(recordedIssue): kind = .issueRecorded issue = EncodedIssue(encoding: recordedIssue, in: eventContext) + _sourceLocation = recordedIssue.sourceLocation case let .valueAttached(attachment): kind = .valueAttached self.attachment = EncodedAttachment(encoding: attachment, in: eventContext) + _sourceLocation = attachment.sourceLocation case .testCaseEnded: if eventContext.test?.isParameterized == false { return nil } kind = .testCaseEnded + case let .testCaseCancelled(skipInfo): + kind = .testCaseCancelled + _sourceLocation = skipInfo.sourceLocation case .testEnded: kind = .testEnded - case .testSkipped: + case let .testSkipped(skipInfo): kind = .testSkipped + _sourceLocation = skipInfo.sourceLocation + case let .testCancelled(skipInfo): + kind = .testCancelled + _sourceLocation = skipInfo.sourceLocation case .runEnded: kind = .runEnded default: diff --git a/Sources/Testing/Parameterization/Test.Case.swift b/Sources/Testing/Parameterization/Test.Case.swift index 375b2f68f..ab9183cf8 100644 --- a/Sources/Testing/Parameterization/Test.Case.swift +++ b/Sources/Testing/Parameterization/Test.Case.swift @@ -233,9 +233,6 @@ extension Test { /// Do not invoke this closure directly. Always use a ``Runner`` to invoke a /// test or test case. var body: @Sendable () async throws -> Void - - /// The task associated with this instance, if any, guarded by a lock. - nonisolated(unsafe) var unsafeCurrentTask = Locked() } /// A type representing a single parameter to a parameterized test function. @@ -288,9 +285,6 @@ extension Test.Case.Argument.ID: Codable {} extension Test.Parameter: Hashable {} extension Test.Case.Argument.ID: Hashable {} -// MARK: - Cancellation - - #if !SWT_NO_SNAPSHOT_TYPES // MARK: - Snapshotting diff --git a/Sources/Testing/Running/Runner.Plan.swift b/Sources/Testing/Running/Runner.Plan.swift index e581bcda8..93af2deab 100644 --- a/Sources/Testing/Running/Runner.Plan.swift +++ b/Sources/Testing/Running/Runner.Plan.swift @@ -196,7 +196,17 @@ extension Runner.Plan { /// The basic "run" action. private static let _runAction = Action.run(options: .init()) + /// Determine what action to perform for a given test by preparing its traits. + /// + /// - Parameters: + /// - test: The test whose action will be determined. + /// + /// - Returns: A tuple containing the action to take for `test` as well as any + /// error that was thrown during trait evaluation. If more than one error + /// was thrown, the first-caught error is returned. private static func _determineAction(for test: Test) async -> (Action, (any Error)?) { + // FIXME: Parallelize this work. Calling `prepare(...)` on all traits and + // evaluating all test arguments should be safely parallelizable. await withTaskGroup(returning: (Action, (any Error)?).self) { taskGroup in taskGroup.addTask { var action = _runAction @@ -209,6 +219,12 @@ extension Runner.Plan { } catch let error as SkipInfo { action = .skip(error) break + } catch is CancellationError { + // Synthesize skip info for this cancellation error. + let sourceContext = SourceContext(backtrace: .current(), sourceLocation: nil) + let skipInfo = SkipInfo(comment: nil, sourceContext: sourceContext) + action = .skip(skipInfo) + break } catch { // Only preserve the first caught error firstCaughtError = firstCaughtError ?? error @@ -281,9 +297,6 @@ extension Runner.Plan { _recursivelyApplyTraits(to: &testGraph) // For each test value, determine the appropriate action for it. - // - // FIXME: Parallelize this work. Calling `prepare(...)` on all traits and - // evaluating all test arguments should be safely parallelizable. testGraph = await testGraph.mapValues { keyPath, test in // Skip any nil test, which implies this node is just a placeholder and // not actual test content. diff --git a/Sources/Testing/Test+Cancellation.swift b/Sources/Testing/Test+Cancellation.swift index 332393afb..dd6f2fd51 100644 --- a/Sources/Testing/Test+Cancellation.swift +++ b/Sources/Testing/Test+Cancellation.swift @@ -13,18 +13,49 @@ /// This protocol is used to abstract away the common implementation of test and /// test case cancellation. protocol TestCancellable: Sendable { - /// The task associated with this instance, if any, guarded by a lock. - var unsafeCurrentTask: Locked { get set } - /// Make an instance of ``Event/Kind`` appropriate for `self`. /// /// - Parameters: /// - skipInfo: The ``SkipInfo`` structure describing the cancellation. /// /// - Returns: An instance of ``Event/Kind`` that describes the cancellation. - func makeCancelledEventKind(with skipInfo: SkipInfo) -> Event.Kind + static func makeCancelledEventKind(with skipInfo: SkipInfo) -> Event.Kind +} + +// MARK: - Tracking the current task + +/// A structure describing a reference to a task that is associated with some +/// ``TestCancellable`` value. +private struct _TaskReference: Sendable { + /// The unsafe underlying reference to the associated task. + private nonisolated(unsafe) var _unsafeCurrentTask = Locked() + + init() { + let unsafeCurrentTask = withUnsafeCurrentTask { $0 } + _unsafeCurrentTask = Locked(rawValue: unsafeCurrentTask) + } + + /// Take this instance's reference to its associated task. + /// + /// - Returns: An `UnsafeCurrentTask` instance, or `nil` if it was already + /// taken or if it was never available. + /// + /// This function consumes the reference to the task. After the first call, + /// subsequent calls on the same instance return `nil`. + func takeUnsafeCurrentTask() -> UnsafeCurrentTask? { + _unsafeCurrentTask.withLock { unsafeCurrentTask in + let result = unsafeCurrentTask + unsafeCurrentTask = nil + return result + } + } } +/// A dictionary of tasks tracked per-task and keyed by types that conform to +/// ``TestCancellable``. +@TaskLocal +private var _currentTaskReferences = [ObjectIdentifier: _TaskReference]() + extension TestCancellable { /// Call a function while the ``unsafeCurrentTask`` property of this instance /// is set to the current task. @@ -39,32 +70,26 @@ extension TestCancellable { /// This function sets the ``unsafeCurrentTask`` property, calls `body`, then /// sets ``unsafeCurrentTask`` back to its previous value. func withCancellationHandling(_ body: () async throws -> R) async rethrows -> R { - // TODO: adopt async withUnsafeCurrentTask() when our deployment target allows - let oldTask = withUnsafeCurrentTask { task in - unsafeCurrentTask.withLock { unsafeCurrentTask in - let oldTask = unsafeCurrentTask - unsafeCurrentTask = task - return oldTask - } - } - defer { - unsafeCurrentTask.withLock { $0 = oldTask } - } - - return try await withTaskCancellationHandler { - try await body() - } onCancel: { - // The current task was cancelled, so cancel the test case or test - // associated with it. - if Test.Case.current != nil { - _ = try? Test.Case.cancel(comment: nil, sourceLocation: nil) - } else if let test = Test.current { - _ = try? _cancel(test, nil, sourceLocation: nil) + var currentTaskReferences = _currentTaskReferences + currentTaskReferences[ObjectIdentifier(Self.self)] = _TaskReference() + return try await $_currentTaskReferences.withValue(currentTaskReferences) { + try await withTaskCancellationHandler { + try await body() + } onCancel: { + // The current task was cancelled, so cancel the test case or test + // associated with it. + if Test.Case.current != nil { + _ = try? Test.Case.cancel(comment: nil, sourceLocation: nil) + } else if let test = Test.current { + _ = try? _cancel(test, comment: nil, sourceLocation: nil) + } } } } } +// MARK: - + /// The common implementation of cancellation for ``Test`` and ``Test/Case``. /// /// - Parameters: @@ -75,24 +100,20 @@ extension TestCancellable { /// attribute the cancellation, if available. /// /// - Throws: An instance of ``SkipInfo`` describing the cancellation. -private func _cancel(_ cancellableValue: (some TestCancellable)?, _ comment: Comment?, sourceLocation: SourceLocation?) throws -> Never { +private func _cancel(_ cancellableValue: T?, comment: Comment?, sourceLocation: SourceLocation?) throws -> Never where T: TestCancellable { let sourceContext = SourceContext(backtrace: .current(), sourceLocation: sourceLocation) let skipInfo = SkipInfo(comment: comment, sourceContext: sourceContext) - if let cancellableValue { + if cancellableValue != nil { // If the current test case is still running, cancel its task and clear its // task property (which signals that it has been cancelled.) - let task = cancellableValue.unsafeCurrentTask.withLock { task in - let result = task - task = nil - return result - } + let task = _currentTaskReferences[ObjectIdentifier(T.self)]?.takeUnsafeCurrentTask() task?.cancel() // If we just cancelled the current test case's task, post a corresponding // event with the relevant skip info. if task != nil { - Event.post(cancellableValue.makeCancelledEventKind(with: skipInfo)) + Event.post(T.makeCancelledEventKind(with: skipInfo)) } } else { // The current task isn't associated with a test case, so just cancel it @@ -101,11 +122,21 @@ private func _cancel(_ cancellableValue: (some TestCancellable)?, _ comment: Com task?.cancel() } - let issue = Issue( - kind: .apiMisused, - comments: ["Attempted to cancel the current test or case, but one is not associated with the current task."] + Array(comment), - sourceContext: sourceContext - ) + let issue = if ExitTest.current != nil { + // Attempted to cancel the test or test case from within an exit test. The + // semantics of such an action aren't yet well-defined. + Issue( + kind: .apiMisused, + comments: ["Attempted to cancel the current test or test case from within an exit test."] + Array(comment), + sourceContext: sourceContext + ) + } else { + Issue( + kind: .apiMisused, + comments: ["Attempted to cancel the current test or test case, but one is not associated with the current task."] + Array(comment), + sourceContext: sourceContext + ) + } issue.record() } @@ -144,6 +175,8 @@ extension Test: TestCancellable { /// test or if it has already finished running, this function throws an error /// but does not attempt to cancel the test a second time. /// + /// - Note: You cannot cancel a test from within the body of an [exit test](doc:exit-testing). + /// /// To cancel the current test case but leave other test cases of the current /// test alone, call ``Test/Case/cancel(_:sourceLocation:)`` instead. /// @@ -152,10 +185,10 @@ extension Test: TestCancellable { /// this function records an issue and cancels the current task. @_spi(Experimental) public static func cancel(_ comment: Comment? = nil, sourceLocation: SourceLocation = #_sourceLocation) throws -> Never { - try _cancel(Test.current, comment, sourceLocation: sourceLocation) + try _cancel(Test.current, comment: comment, sourceLocation: sourceLocation) } - func makeCancelledEventKind(with skipInfo: SkipInfo) -> Event.Kind { + static func makeCancelledEventKind(with skipInfo: SkipInfo) -> Event.Kind { .testCancelled(skipInfo) } } @@ -180,11 +213,11 @@ extension Test.Case: TestCancellable { if let test = Test.current, !test.isParameterized { // The current test is not parameterized, so cancel the whole test rather // than just the test case. - try _cancel(test, comment, sourceLocation: sourceLocation) + try _cancel(test, comment: comment, sourceLocation: sourceLocation) } // Cancel the current test case (if it's nil, that's the API misuse path.) - try _cancel(Test.Case.current, comment, sourceLocation: sourceLocation) + try _cancel(Test.Case.current, comment: comment, sourceLocation: sourceLocation) } /// Cancel the current test case. @@ -216,18 +249,20 @@ extension Test.Case: TestCancellable { /// function throws an error but does not attempt to cancel the test case a /// second time. /// + /// - Note: You cannot cancel a test case from within the body of an [exit test](doc:exit-testing). + /// /// To cancel all test cases in the current test, call /// ``Test/cancel(_:sourceLocation:)`` instead. /// /// - Important: If the current task is not associated with a test case (for /// example, because it was created with [`Task.detached(name:priority:operation:)`](https://developer.apple.com/documentation/swift/task/detached(name:priority:operation:)-795w1)) /// this function records an issue and cancels the current task. - @_spi(Experimental) + //@_spi(Experimental) public static func cancel(_ comment: Comment? = nil, sourceLocation: SourceLocation = #_sourceLocation) throws -> Never { try cancel(comment: comment, sourceLocation: sourceLocation) } - func makeCancelledEventKind(with skipInfo: SkipInfo) -> Event.Kind { + static func makeCancelledEventKind(with skipInfo: SkipInfo) -> Event.Kind { .testCaseCancelled(skipInfo) } } diff --git a/Sources/Testing/Test.swift b/Sources/Testing/Test.swift index 52670ffba..5f2ac2406 100644 --- a/Sources/Testing/Test.swift +++ b/Sources/Testing/Test.swift @@ -201,9 +201,6 @@ public struct Test: Sendable { @_spi(ForToolsIntegrationOnly) public var isSynthesized: Bool = false - /// The task associated with this test, if any, guarded by a lock. - nonisolated(unsafe) var unsafeCurrentTask = Locked() - /// Initialize an instance of this type representing a test suite type. init( displayName: String? = nil, From 5e6880a4682296e1d0419b87e896308da65ba2f4 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Mon, 25 Aug 2025 20:55:57 -0400 Subject: [PATCH 09/24] Add more tests --- Sources/Testing/Test+Cancellation.swift | 5 +++-- Tests/TestingTests/TestCancellationTests.swift | 16 +++++++++++++++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/Sources/Testing/Test+Cancellation.swift b/Sources/Testing/Test+Cancellation.swift index dd6f2fd51..8c30a2559 100644 --- a/Sources/Testing/Test+Cancellation.swift +++ b/Sources/Testing/Test+Cancellation.swift @@ -67,8 +67,9 @@ extension TestCancellable { /// /// - Throws: Whatever is thrown by `body`. /// - /// This function sets the ``unsafeCurrentTask`` property, calls `body`, then - /// sets ``unsafeCurrentTask`` back to its previous value. + /// This function sets up a task cancellation handler and calls `body`. If + /// the current task, test, or test case is cancelled, it records a + /// corresponding cancellation event. func withCancellationHandling(_ body: () async throws -> R) async rethrows -> R { var currentTaskReferences = _currentTaskReferences currentTaskReferences[ObjectIdentifier(Self.self)] = _TaskReference() diff --git a/Tests/TestingTests/TestCancellationTests.swift b/Tests/TestingTests/TestCancellationTests.swift index 519eaa85f..43c8e3b15 100644 --- a/Tests/TestingTests/TestCancellationTests.swift +++ b/Tests/TestingTests/TestCancellationTests.swift @@ -107,16 +107,30 @@ struct `Test cancellation tests` { } struct CancelledTrait: TestTrait { + var cancelsTask = false + func prepare(for test: Test) async throws { + if cancelsTask { + withUnsafeCurrentTask { $0?.cancel() } + throw CancellationError() + } try Test.cancel("Cancelled from trait") } } - @Test func `Cancelling a test case while evaluating traits skips the test`() async { + @Test func `Cancelling a test while evaluating traits skips the test`() async { await testCancellation(testSkipped: 1) { configuration in await Test(CancelledTrait()) { Issue.record("Recorded an issue!") }.run(configuration: configuration) } } + + @Test func `Cancelling the current task while evaluating traits skips the test`() async { + await testCancellation(testSkipped: 1) { configuration in + await Test(CancelledTrait(cancelsTask: true)) { + Issue.record("Recorded an issue!") + }.run(configuration: configuration) + } + } } From 344da945c90a637d8e4a813a85983dd16bcf92d0 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Mon, 25 Aug 2025 21:28:41 -0400 Subject: [PATCH 10/24] More tests, refactor cancel() a bit --- Sources/Testing/Test+Cancellation.swift | 93 ++++++++++--------- .../TestingTests/TestCancellationTests.swift | 45 +++++++++ 2 files changed, 92 insertions(+), 46 deletions(-) diff --git a/Sources/Testing/Test+Cancellation.swift b/Sources/Testing/Test+Cancellation.swift index 8c30a2559..bb28aeb2f 100644 --- a/Sources/Testing/Test+Cancellation.swift +++ b/Sources/Testing/Test+Cancellation.swift @@ -13,6 +13,21 @@ /// This protocol is used to abstract away the common implementation of test and /// test case cancellation. protocol TestCancellable: Sendable { + /// Cancel the current instance of this type. + /// + /// - Parameters: + /// - comment: A comment describing why you are cancelling the test. + /// - sourceContext: The source context to which the testing library will + /// attribute the cancellation. + /// + /// - Throws: An error indicating that the current instance of this type has + /// been cancelled. + /// + /// Note that the public ``Test/cancel(_:sourceLocation:)`` function has a + /// different signature and accepts a source location rather than a source + /// context value. + static func cancel(comment: Comment?, sourceContext: SourceContext) throws -> Never + /// Make an instance of ``Event/Kind`` appropriate for `self`. /// /// - Parameters: @@ -79,11 +94,8 @@ extension TestCancellable { } onCancel: { // The current task was cancelled, so cancel the test case or test // associated with it. - if Test.Case.current != nil { - _ = try? Test.Case.cancel(comment: nil, sourceLocation: nil) - } else if let test = Test.current { - _ = try? _cancel(test, comment: nil, sourceLocation: nil) - } + let sourceContext = SourceContext(backtrace: .current(), sourceLocation: nil) + _ = try? Self.cancel(comment: nil, sourceContext: sourceContext) } } } @@ -97,12 +109,11 @@ extension TestCancellable { /// - cancellableValue: The test or test case to cancel, or `nil` if neither /// is set and we need fallback handling. /// - comment: A comment describing why you are cancelling the test/case. -/// - sourceLocation: The source location to which the testing library will -/// attribute the cancellation, if available. +/// - sourceContext: The source context to which the testing library will +/// attribute the cancellation. /// /// - Throws: An instance of ``SkipInfo`` describing the cancellation. -private func _cancel(_ cancellableValue: T?, comment: Comment?, sourceLocation: SourceLocation?) throws -> Never where T: TestCancellable { - let sourceContext = SourceContext(backtrace: .current(), sourceLocation: sourceLocation) +private func _cancel(_ cancellableValue: T?, comment: Comment?, sourceContext: SourceContext) throws -> Never where T: TestCancellable { let skipInfo = SkipInfo(comment: comment, sourceContext: sourceContext) if cancellableValue != nil { @@ -123,21 +134,18 @@ private func _cancel(_ cancellableValue: T?, comment: Comment?, sourceLocatio task?.cancel() } - let issue = if ExitTest.current != nil { + let comment: Comment = if ExitTest.current != nil { // Attempted to cancel the test or test case from within an exit test. The // semantics of such an action aren't yet well-defined. - Issue( - kind: .apiMisused, - comments: ["Attempted to cancel the current test or test case from within an exit test."] + Array(comment), - sourceContext: sourceContext - ) + "Attempted to cancel the current test or test case from within an exit test." } else { - Issue( - kind: .apiMisused, - comments: ["Attempted to cancel the current test or test case, but one is not associated with the current task."] + Array(comment), - sourceContext: sourceContext - ) + "Attempted to cancel the current test or test case, but one is not associated with the current task." } + let issue = Issue( + kind: .apiMisused, + comments: CollectionOfOne(comment) + Array(comment), + sourceContext: sourceContext + ) issue.record() } @@ -186,7 +194,12 @@ extension Test: TestCancellable { /// this function records an issue and cancels the current task. @_spi(Experimental) public static func cancel(_ comment: Comment? = nil, sourceLocation: SourceLocation = #_sourceLocation) throws -> Never { - try _cancel(Test.current, comment: comment, sourceLocation: sourceLocation) + let sourceContext = SourceContext(backtrace: .current(), sourceLocation: sourceLocation) + try Self.cancel(comment: comment, sourceContext: sourceContext) + } + + static func cancel(comment: Comment?, sourceContext: SourceContext) throws -> Never { + try _cancel(Test.current, comment: comment, sourceContext: sourceContext) } static func makeCancelledEventKind(with skipInfo: SkipInfo) -> Event.Kind { @@ -197,30 +210,6 @@ extension Test: TestCancellable { // MARK: - Test case cancellation extension Test.Case: TestCancellable { - /// The implementation of ``cancel(_:sourceLocation:)``, but able to take a - /// `nil` value as its `sourceLocation` argument. - /// - /// - Parameters: - /// - comment: A comment describing why you are cancelling the test. - /// - sourceLocation: The source location to which the testing library will - /// attribute the cancellation. - /// - /// - Throws: An error indicating that the current test case has been - /// cancelled. - /// - /// This overload of `cancel()` is factored out so we can call it with a `nil` - /// source location in ``withCancellationHandling(_:)``. - fileprivate static func cancel(comment: Comment?, sourceLocation: SourceLocation?) throws -> Never { - if let test = Test.current, !test.isParameterized { - // The current test is not parameterized, so cancel the whole test rather - // than just the test case. - try _cancel(test, comment: comment, sourceLocation: sourceLocation) - } - - // Cancel the current test case (if it's nil, that's the API misuse path.) - try _cancel(Test.Case.current, comment: comment, sourceLocation: sourceLocation) - } - /// Cancel the current test case. /// /// - Parameters: @@ -260,7 +249,19 @@ extension Test.Case: TestCancellable { /// this function records an issue and cancels the current task. //@_spi(Experimental) public static func cancel(_ comment: Comment? = nil, sourceLocation: SourceLocation = #_sourceLocation) throws -> Never { - try cancel(comment: comment, sourceLocation: sourceLocation) + let sourceContext = SourceContext(backtrace: .current(), sourceLocation: sourceLocation) + try Self.cancel(comment: comment, sourceContext: sourceContext) + } + + static func cancel(comment: Comment?, sourceContext: SourceContext) throws -> Never { + if let test = Test.current, !test.isParameterized { + // The current test is not parameterized, so cancel the whole test rather + // than just the test case. + try _cancel(test, comment: comment, sourceContext: sourceContext) + } + + // Cancel the current test case (if it's nil, that's the API misuse path.) + try _cancel(Test.Case.current, comment: comment, sourceContext: sourceContext) } static func makeCancelledEventKind(with skipInfo: SkipInfo) -> Event.Kind { diff --git a/Tests/TestingTests/TestCancellationTests.swift b/Tests/TestingTests/TestCancellationTests.swift index 43c8e3b15..0763e9ea4 100644 --- a/Tests/TestingTests/TestCancellationTests.swift +++ b/Tests/TestingTests/TestCancellationTests.swift @@ -34,6 +34,9 @@ struct `Test cancellation tests` { break } } +#if !SWT_NO_EXIT_TESTS + configuration.exitTestHandler = ExitTest.handlerForEntryPoint() +#endif await body(configuration) } } @@ -133,4 +136,46 @@ struct `Test cancellation tests` { }.run(configuration: configuration) } } + +#if !SWT_NO_EXIT_TESTS + @Test func `Cancelling from within an exit test records an issue`() async { + await testCancellation(issueRecorded: 1) { configuration in + await Test { + await #expect(processExitsWith: .success) { + _ = try? Test.cancel("Cancelled test") + } + }.run(configuration: configuration) + } + } +#endif +} + +#if canImport(XCTest) +import XCTest + +final class TestCancellationTests: XCTestCase { + func testCancellationFromBackgroundTask() async { + let testCancelled = expectation(description: "Test cancelled") + testCancelled.isInverted = true + + let issueRecorded = expectation(description: "Issue recorded") + + var configuration = Configuration() + configuration.eventHandler = { event, _ in + if case .testCancelled = event.kind { + testCancelled.fulfill() + } else if case .issueRecorded = event.kind { + issueRecorded.fulfill() + } + } + + await Test { + await Task.detached { + _ = try? Test.cancel("Cancelled test") + }.value + }.run(configuration: configuration) + + await fulfillment(of: [testCancelled, issueRecorded], timeout: 0.0) + } } +#endif From ef18699f06236f2de52642596998374e6ed0f633 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Mon, 25 Aug 2025 21:32:28 -0400 Subject: [PATCH 11/24] Restore @spi attribute --- Sources/Testing/Test+Cancellation.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/Testing/Test+Cancellation.swift b/Sources/Testing/Test+Cancellation.swift index bb28aeb2f..72d3a4832 100644 --- a/Sources/Testing/Test+Cancellation.swift +++ b/Sources/Testing/Test+Cancellation.swift @@ -247,7 +247,7 @@ extension Test.Case: TestCancellable { /// - Important: If the current task is not associated with a test case (for /// example, because it was created with [`Task.detached(name:priority:operation:)`](https://developer.apple.com/documentation/swift/task/detached(name:priority:operation:)-795w1)) /// this function records an issue and cancels the current task. - //@_spi(Experimental) + @_spi(Experimental) public static func cancel(_ comment: Comment? = nil, sourceLocation: SourceLocation = #_sourceLocation) throws -> Never { let sourceContext = SourceContext(backtrace: .current(), sourceLocation: sourceLocation) try Self.cancel(comment: comment, sourceContext: sourceContext) From f0b6e9e633ad91a30d9946bc9f3b12d0eb995bb4 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Mon, 25 Aug 2025 21:44:02 -0400 Subject: [PATCH 12/24] Add new event kinds to Snapshot for Xcode 16 compatibility --- .../Testing/ABI/ABI.Record+Streaming.swift | 9 +----- Sources/Testing/Events/Event.swift | 30 ++++++++++++++++++- 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/Sources/Testing/ABI/ABI.Record+Streaming.swift b/Sources/Testing/ABI/ABI.Record+Streaming.swift index ab22714ae..1aa1362ec 100644 --- a/Sources/Testing/ABI/ABI.Record+Streaming.swift +++ b/Sources/Testing/ABI/ABI.Record+Streaming.swift @@ -47,18 +47,11 @@ extension ABI.Xcode16 { forwardingTo eventHandler: @escaping @Sendable (_ recordJSON: UnsafeRawBufferPointer) -> Void ) -> Event.Handler { return { event, context in - switch event.kind { - case .testDiscovered: + if case .testDiscovered = event.kind { // Discard events of this kind rather than forwarding them to avoid a // crash in Xcode 16 (which does not expect any events to occur before // .runStarted.) return - case .testCancelled, .testCaseCancelled: - // Discard these events as Xcode 16 does not know how to handle them and - // may crash if they arrive during trait evaluation. - return - default: - break } struct EventAndContextSnapshot: Codable { diff --git a/Sources/Testing/Events/Event.swift b/Sources/Testing/Events/Event.swift index 00f81fe5b..d8daa3e89 100644 --- a/Sources/Testing/Events/Event.swift +++ b/Sources/Testing/Events/Event.swift @@ -419,6 +419,18 @@ extension Event.Kind { /// A test case ended. case testCaseEnded + /// A test case was cancelled. + /// + /// - Parameters: + /// - skipInfo: A ``SkipInfo`` with details about the cancelled test case. + /// + /// This event is generated by a call to ``Test/Case/cancel(_:sourceLocation:)``. + /// + /// The test case that was cancelled is contained in the ``Event/Context`` + /// instance that was passed to the event handler along with this event. + @_spi(Experimental) + indirect case testCaseCancelled(_ skipInfo: SkipInfo) + /// An expectation was checked with `#expect()` or `#require()`. /// /// - Parameters: @@ -455,6 +467,18 @@ extension Event.Kind { /// - skipInfo: A ``SkipInfo`` containing details about this skipped test. indirect case testSkipped(_ skipInfo: SkipInfo) + /// A test was cancelled. + /// + /// - Parameters: + /// - skipInfo: A ``SkipInfo`` with details about the cancelled test. + /// + /// This event is generated by a call to ``Test/cancel(_:sourceLocation:)``. + /// + /// The test that was cancelled is contained in the ``Event/Context`` + /// instance that was passed to the event handler along with this event. + @_spi(Experimental) + indirect case testCancelled(_ skipInfo: SkipInfo) + /// A step in the runner plan ended. /// /// - Parameters: @@ -503,6 +527,8 @@ extension Event.Kind { self = .testCaseStarted case .testCaseEnded: self = .testCaseEnded + case let .testCaseCancelled(skipInfo): + self = .testCaseCancelled(skipInfo) case let .expectationChecked(expectation): let expectationSnapshot = Expectation.Snapshot(snapshotting: expectation) self = Snapshot.expectationChecked(expectationSnapshot) @@ -512,8 +538,10 @@ extension Event.Kind { self = .valueAttached case .testEnded: self = .testEnded - case let .testCancelled(skipInfo), let .testSkipped(skipInfo), let .testCaseCancelled(skipInfo): + case let .testSkipped(skipInfo): self = .testSkipped(skipInfo) + case let .testCancelled(skipInfo): + self = .testCancelled(skipInfo) case .planStepEnded: self = .planStepEnded case let .iterationEnded(index): From 87fcdd577f3b6d8e772c06047e8328cd48a94f65 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Mon, 25 Aug 2025 22:26:36 -0400 Subject: [PATCH 13/24] Make Xcode 16 report skipped tests where possible --- .../Testing/ABI/ABI.Record+Streaming.swift | 1 + Sources/Testing/Test+Cancellation.swift | 27 +++++++----- .../TestingTests/TestCancellationTests.swift | 44 ++++++++++++------- 3 files changed, 44 insertions(+), 28 deletions(-) diff --git a/Sources/Testing/ABI/ABI.Record+Streaming.swift b/Sources/Testing/ABI/ABI.Record+Streaming.swift index 1aa1362ec..5421f2a81 100644 --- a/Sources/Testing/ABI/ABI.Record+Streaming.swift +++ b/Sources/Testing/ABI/ABI.Record+Streaming.swift @@ -62,6 +62,7 @@ extension ABI.Xcode16 { event: Event.Snapshot(snapshotting: event), eventContext: Event.Context.Snapshot(snapshotting: context) ) + try? JSON.withEncoding(of: snapshot) { eventAndContextJSON in eventAndContextJSON.withUnsafeBytes { eventAndContextJSON in eventHandler(eventAndContextJSON) diff --git a/Sources/Testing/Test+Cancellation.swift b/Sources/Testing/Test+Cancellation.swift index 72d3a4832..809a09928 100644 --- a/Sources/Testing/Test+Cancellation.swift +++ b/Sources/Testing/Test+Cancellation.swift @@ -28,7 +28,8 @@ protocol TestCancellable: Sendable { /// context value. static func cancel(comment: Comment?, sourceContext: SourceContext) throws -> Never - /// Make an instance of ``Event/Kind`` appropriate for `self`. + /// Make an instance of ``Event/Kind`` appropriate for an instance of this + /// type. /// /// - Parameters: /// - skipInfo: The ``SkipInfo`` structure describing the cancellation. @@ -108,12 +109,13 @@ extension TestCancellable { /// - Parameters: /// - cancellableValue: The test or test case to cancel, or `nil` if neither /// is set and we need fallback handling. +/// - testAndTestCase: The test and test case to use when posting an event. /// - comment: A comment describing why you are cancelling the test/case. /// - sourceContext: The source context to which the testing library will /// attribute the cancellation. /// /// - Throws: An instance of ``SkipInfo`` describing the cancellation. -private func _cancel(_ cancellableValue: T?, comment: Comment?, sourceContext: SourceContext) throws -> Never where T: TestCancellable { +private func _cancel(_ cancellableValue: T?, for testAndTestCase: (Test?, Test.Case?), comment: Comment?, sourceContext: SourceContext) throws -> Never where T: TestCancellable { let skipInfo = SkipInfo(comment: comment, sourceContext: sourceContext) if cancellableValue != nil { @@ -125,7 +127,7 @@ private func _cancel(_ cancellableValue: T?, comment: Comment?, sourceContext // If we just cancelled the current test case's task, post a corresponding // event with the relevant skip info. if task != nil { - Event.post(T.makeCancelledEventKind(with: skipInfo)) + Event.post(T.makeCancelledEventKind(with: skipInfo), for: testAndTestCase) } } else { // The current task isn't associated with a test case, so just cancel it @@ -199,7 +201,8 @@ extension Test: TestCancellable { } static func cancel(comment: Comment?, sourceContext: SourceContext) throws -> Never { - try _cancel(Test.current, comment: comment, sourceContext: sourceContext) + let test = Test.current + try _cancel(test, for: (test, nil), comment: comment, sourceContext: sourceContext) } static func makeCancelledEventKind(with skipInfo: SkipInfo) -> Event.Kind { @@ -254,14 +257,16 @@ extension Test.Case: TestCancellable { } static func cancel(comment: Comment?, sourceContext: SourceContext) throws -> Never { - if let test = Test.current, !test.isParameterized { - // The current test is not parameterized, so cancel the whole test rather - // than just the test case. - try _cancel(test, comment: comment, sourceContext: sourceContext) - } + let test = Test.current + let testCase = Test.Case.current - // Cancel the current test case (if it's nil, that's the API misuse path.) - try _cancel(Test.Case.current, comment: comment, sourceContext: sourceContext) + do { + // Cancel the current test case (if it's nil, that's the API misuse path.) + try _cancel(testCase, for: (test, testCase), comment: comment, sourceContext: sourceContext) + } catch _ where test?.isParameterized == false { + // The current test is not parameterized, so cancel the whole test too. + try _cancel(test, for: (test, nil), comment: comment, sourceContext: sourceContext) + } } static func makeCancelledEventKind(with skipInfo: SkipInfo) -> Event.Kind { diff --git a/Tests/TestingTests/TestCancellationTests.swift b/Tests/TestingTests/TestCancellationTests.swift index 0763e9ea4..5a09b8e32 100644 --- a/Tests/TestingTests/TestCancellationTests.swift +++ b/Tests/TestingTests/TestCancellationTests.swift @@ -10,7 +10,7 @@ @testable @_spi(Experimental) @_spi(ForToolsIntegrationOnly) import Testing -struct `Test cancellation tests` { +@Suite(.serialized) struct `Test cancellation tests` { func testCancellation(testCancelled: Int = 0, testSkipped: Int = 0, testCaseCancelled: Int = 0, issueRecorded: Int = 0, _ body: @Sendable (Configuration) async -> Void) async { await confirmation("Test cancelled", expectedCount: testCancelled) { testCancelled in await confirmation("Test skipped", expectedCount: testSkipped) { testSkipped in @@ -45,7 +45,7 @@ struct `Test cancellation tests` { } @Test func `Cancelling a test`() async { - await testCancellation(testCancelled: 1) { configuration in + await testCancellation(testCancelled: 1, testCaseCancelled: 1) { configuration in await Test { try Test.cancel("Cancelled test") }.run(configuration: configuration) @@ -53,7 +53,7 @@ struct `Test cancellation tests` { } @Test func `Cancelling a non-parameterized test via Test.Case.cancel()`() async { - await testCancellation(testCancelled: 1) { configuration in + await testCancellation(testCancelled: 1, testCaseCancelled: 1) { configuration in await Test { try Test.Case.cancel("Cancelled test") }.run(configuration: configuration) @@ -85,7 +85,7 @@ struct `Test cancellation tests` { } @Test func `Cancelling a test by cancelling its task (throwing)`() async { - await testCancellation(testCancelled: 1) { configuration in + await testCancellation(testCancelled: 1, testCaseCancelled: 1) { configuration in await Test { withUnsafeCurrentTask { $0?.cancel() } try Task.checkCancellation() @@ -94,7 +94,7 @@ struct `Test cancellation tests` { } @Test func `Cancelling a test by cancelling its task (returning)`() async { - await testCancellation(testCancelled: 1) { configuration in + await testCancellation(testCancelled: 1, testCaseCancelled: 1) { configuration in await Test { withUnsafeCurrentTask { $0?.cancel() } }.run(configuration: configuration) @@ -109,18 +109,6 @@ struct `Test cancellation tests` { } } - struct CancelledTrait: TestTrait { - var cancelsTask = false - - func prepare(for test: Test) async throws { - if cancelsTask { - withUnsafeCurrentTask { $0?.cancel() } - throw CancellationError() - } - try Test.cancel("Cancelled from trait") - } - } - @Test func `Cancelling a test while evaluating traits skips the test`() async { await testCancellation(testSkipped: 1) { configuration in await Test(CancelledTrait()) { @@ -179,3 +167,25 @@ final class TestCancellationTests: XCTestCase { } } #endif + +// MARK: - Fixtures + +struct CancelledTrait: TestTrait { + var cancelsTask = false + + func prepare(for test: Test) async throws { + if cancelsTask { + withUnsafeCurrentTask { $0?.cancel() } + throw CancellationError() + } + try Test.cancel("Cancelled from trait") + } +} + +#if !SWT_NO_SNAPSHOT_TYPES +struct `Shows as skipped in Xcode 16` { + @Test func `Cancelled test`() throws { + try Test.cancel("This test should appear cancelled/skipped") + } +} +#endif From b32e5029efaf4e33adb8a8d2678c8fe601579611 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Mon, 25 Aug 2025 23:07:08 -0400 Subject: [PATCH 14/24] Lazily gather backtraces where possible --- .../Testing/ABI/ABI.Record+Streaming.swift | 1 - Sources/Testing/Test+Cancellation.swift | 34 ++++++++++++------- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/Sources/Testing/ABI/ABI.Record+Streaming.swift b/Sources/Testing/ABI/ABI.Record+Streaming.swift index 5421f2a81..1aa1362ec 100644 --- a/Sources/Testing/ABI/ABI.Record+Streaming.swift +++ b/Sources/Testing/ABI/ABI.Record+Streaming.swift @@ -62,7 +62,6 @@ extension ABI.Xcode16 { event: Event.Snapshot(snapshotting: event), eventContext: Event.Context.Snapshot(snapshotting: context) ) - try? JSON.withEncoding(of: snapshot) { eventAndContextJSON in eventAndContextJSON.withUnsafeBytes { eventAndContextJSON in eventHandler(eventAndContextJSON) diff --git a/Sources/Testing/Test+Cancellation.swift b/Sources/Testing/Test+Cancellation.swift index 809a09928..adc25677e 100644 --- a/Sources/Testing/Test+Cancellation.swift +++ b/Sources/Testing/Test+Cancellation.swift @@ -26,7 +26,7 @@ protocol TestCancellable: Sendable { /// Note that the public ``Test/cancel(_:sourceLocation:)`` function has a /// different signature and accepts a source location rather than a source /// context value. - static func cancel(comment: Comment?, sourceContext: SourceContext) throws -> Never + static func cancel(comment: Comment?, sourceContext: @autoclosure () -> SourceContext) throws -> Never /// Make an instance of ``Event/Kind`` appropriate for an instance of this /// type. @@ -95,8 +95,10 @@ extension TestCancellable { } onCancel: { // The current task was cancelled, so cancel the test case or test // associated with it. - let sourceContext = SourceContext(backtrace: .current(), sourceLocation: nil) - _ = try? Self.cancel(comment: nil, sourceContext: sourceContext) + _ = try? Self.cancel( + comment: nil, + sourceContext: SourceContext(backtrace: .current(), sourceLocation: nil) + ) } } } @@ -115,8 +117,8 @@ extension TestCancellable { /// attribute the cancellation. /// /// - Throws: An instance of ``SkipInfo`` describing the cancellation. -private func _cancel(_ cancellableValue: T?, for testAndTestCase: (Test?, Test.Case?), comment: Comment?, sourceContext: SourceContext) throws -> Never where T: TestCancellable { - let skipInfo = SkipInfo(comment: comment, sourceContext: sourceContext) +private func _cancel(_ cancellableValue: T?, for testAndTestCase: (Test?, Test.Case?), comment: Comment?, sourceContext: @autoclosure () -> SourceContext) throws -> Never where T: TestCancellable { + var skipInfo = SkipInfo(comment: comment, sourceContext: .init(backtrace: nil, sourceLocation: nil)) if cancellableValue != nil { // If the current test case is still running, cancel its task and clear its @@ -127,6 +129,7 @@ private func _cancel(_ cancellableValue: T?, for testAndTestCase: (Test?, Tes // If we just cancelled the current test case's task, post a corresponding // event with the relevant skip info. if task != nil { + skipInfo.sourceContext = sourceContext() Event.post(T.makeCancelledEventKind(with: skipInfo), for: testAndTestCase) } } else { @@ -146,7 +149,7 @@ private func _cancel(_ cancellableValue: T?, for testAndTestCase: (Test?, Tes let issue = Issue( kind: .apiMisused, comments: CollectionOfOne(comment) + Array(comment), - sourceContext: sourceContext + sourceContext: sourceContext() ) issue.record() } @@ -196,13 +199,15 @@ extension Test: TestCancellable { /// this function records an issue and cancels the current task. @_spi(Experimental) public static func cancel(_ comment: Comment? = nil, sourceLocation: SourceLocation = #_sourceLocation) throws -> Never { - let sourceContext = SourceContext(backtrace: .current(), sourceLocation: sourceLocation) - try Self.cancel(comment: comment, sourceContext: sourceContext) + try Self.cancel( + comment: comment, + sourceContext: SourceContext(backtrace: .current(), sourceLocation: sourceLocation) + ) } - static func cancel(comment: Comment?, sourceContext: SourceContext) throws -> Never { + static func cancel(comment: Comment?, sourceContext: @autoclosure () -> SourceContext) throws -> Never { let test = Test.current - try _cancel(test, for: (test, nil), comment: comment, sourceContext: sourceContext) + try _cancel(test, for: (test, nil), comment: comment, sourceContext: sourceContext()) } static func makeCancelledEventKind(with skipInfo: SkipInfo) -> Event.Kind { @@ -252,13 +257,16 @@ extension Test.Case: TestCancellable { /// this function records an issue and cancels the current task. @_spi(Experimental) public static func cancel(_ comment: Comment? = nil, sourceLocation: SourceLocation = #_sourceLocation) throws -> Never { - let sourceContext = SourceContext(backtrace: .current(), sourceLocation: sourceLocation) - try Self.cancel(comment: comment, sourceContext: sourceContext) + try Self.cancel( + comment: comment, + sourceContext: SourceContext(backtrace: .current(), sourceLocation: sourceLocation) + ) } - static func cancel(comment: Comment?, sourceContext: SourceContext) throws -> Never { + static func cancel(comment: Comment?, sourceContext: @autoclosure () -> SourceContext) throws -> Never { let test = Test.current let testCase = Test.Case.current + let sourceContext = sourceContext() // evaluated twice, avoid laziness do { // Cancel the current test case (if it's nil, that's the API misuse path.) From 7371b60ed4b09eff6827f92c3c9e81d56166d25e Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Tue, 26 Aug 2025 09:33:09 -0400 Subject: [PATCH 15/24] Deduplicate attachment JSON source location field --- Sources/Testing/ABI/Encoded/ABI.EncodedAttachment.swift | 8 -------- Sources/Testing/ExitTests/ExitTest.swift | 2 +- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/Sources/Testing/ABI/Encoded/ABI.EncodedAttachment.swift b/Sources/Testing/ABI/Encoded/ABI.EncodedAttachment.swift index 9275da1ed..a2400f9bf 100644 --- a/Sources/Testing/ABI/Encoded/ABI.EncodedAttachment.swift +++ b/Sources/Testing/ABI/Encoded/ABI.EncodedAttachment.swift @@ -38,12 +38,6 @@ extension ABI { /// - Warning: Inline attachment content is not yet part of the JSON schema. var _bytes: Bytes? - /// The source location where this attachment was created. - /// - /// - Warning: Attachment source locations are not yet part of the JSON - /// schema. - var _sourceLocation: SourceLocation? - init(encoding attachment: borrowing Attachment, in eventContext: borrowing Event.Context) { path = attachment.fileSystemPath @@ -55,8 +49,6 @@ extension ABI { return Bytes(rawValue: [UInt8](bytes)) } } - - _sourceLocation = attachment.sourceLocation } } diff --git a/Sources/Testing/ExitTests/ExitTest.swift b/Sources/Testing/ExitTests/ExitTest.swift index 5eb006b03..c716e421c 100644 --- a/Sources/Testing/ExitTests/ExitTest.swift +++ b/Sources/Testing/ExitTests/ExitTest.swift @@ -1072,7 +1072,7 @@ extension ExitTest { } issueCopy.record() } else if let attachment = event.attachment { - Attachment.record(attachment, sourceLocation: attachment._sourceLocation!) + Attachment.record(attachment, sourceLocation: event._sourceLocation!) } } From 6650c9c051852524290c5144e9c9bdf6198fad18 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Tue, 26 Aug 2025 09:46:04 -0400 Subject: [PATCH 16/24] Fix duplicated comment --- Sources/Testing/Test+Cancellation.swift | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/Sources/Testing/Test+Cancellation.swift b/Sources/Testing/Test+Cancellation.swift index adc25677e..026fff1e6 100644 --- a/Sources/Testing/Test+Cancellation.swift +++ b/Sources/Testing/Test+Cancellation.swift @@ -67,7 +67,7 @@ private struct _TaskReference: Sendable { } } -/// A dictionary of tasks tracked per-task and keyed by types that conform to +/// A dictionary of tracked tasks, keyed by types that conform to /// ``TestCancellable``. @TaskLocal private var _currentTaskReferences = [ObjectIdentifier: _TaskReference]() @@ -139,18 +139,15 @@ private func _cancel(_ cancellableValue: T?, for testAndTestCase: (Test?, Tes task?.cancel() } - let comment: Comment = if ExitTest.current != nil { + var comments: [Comment] = if ExitTest.current != nil { // Attempted to cancel the test or test case from within an exit test. The // semantics of such an action aren't yet well-defined. - "Attempted to cancel the current test or test case from within an exit test." + ["Attempted to cancel the current test or test case from within an exit test."] } else { - "Attempted to cancel the current test or test case, but one is not associated with the current task." + ["Attempted to cancel the current test or test case, but one is not associated with the current task."] } - let issue = Issue( - kind: .apiMisused, - comments: CollectionOfOne(comment) + Array(comment), - sourceContext: sourceContext() - ) + comments.append(comment) + let issue = Issue(kind: .apiMisused, comments: comments, sourceContext: sourceContext()) issue.record() } From ef609ab13afc04f99696b004b2d341972b8ce4ce Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Tue, 26 Aug 2025 11:52:35 -0400 Subject: [PATCH 17/24] Fix typo --- Sources/Testing/Test+Cancellation.swift | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Sources/Testing/Test+Cancellation.swift b/Sources/Testing/Test+Cancellation.swift index 026fff1e6..53b05c6b3 100644 --- a/Sources/Testing/Test+Cancellation.swift +++ b/Sources/Testing/Test+Cancellation.swift @@ -146,7 +146,9 @@ private func _cancel(_ cancellableValue: T?, for testAndTestCase: (Test?, Tes } else { ["Attempted to cancel the current test or test case, but one is not associated with the current task."] } - comments.append(comment) + if let comment { + comments.append(comment) + } let issue = Issue(kind: .apiMisused, comments: comments, sourceContext: sourceContext()) issue.record() } From 2da8459cceed2d11fb4d6cf652a8d6b7a594ce89 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Tue, 26 Aug 2025 12:36:12 -0400 Subject: [PATCH 18/24] Plumb through exit test support properly --- .../ABI/Encoded/ABI.EncodedEvent.swift | 9 +++ Sources/Testing/ExitTests/ExitTest.swift | 21 ++++--- Sources/Testing/Issues/Issue+Recording.swift | 10 +-- Sources/Testing/Test+Cancellation.swift | 63 +++++++++++-------- .../TestingTests/TestCancellationTests.swift | 20 +++++- 5 files changed, 81 insertions(+), 42 deletions(-) diff --git a/Sources/Testing/ABI/Encoded/ABI.EncodedEvent.swift b/Sources/Testing/ABI/Encoded/ABI.EncodedEvent.swift index 09bc41a63..b7a94c5f4 100644 --- a/Sources/Testing/ABI/Encoded/ABI.EncodedEvent.swift +++ b/Sources/Testing/ABI/Encoded/ABI.EncodedEvent.swift @@ -66,6 +66,11 @@ extension ABI { /// - Warning: Test cases are not yet part of the JSON schema. var _testCase: EncodedTestCase? + /// The comments the test author provided for this event, if any. + /// + /// - Warning: Comments at this level are not yet part of the JSON schema. + var _comments: [String]? + /// A source location associated with this event, if any. /// /// - Warning: Source locations at this level of the JSON schema are not yet @@ -86,6 +91,7 @@ extension ABI { case let .issueRecorded(recordedIssue): kind = .issueRecorded issue = EncodedIssue(encoding: recordedIssue, in: eventContext) + _comments = recordedIssue.comments.map(\.rawValue) _sourceLocation = recordedIssue.sourceLocation case let .valueAttached(attachment): kind = .valueAttached @@ -98,14 +104,17 @@ extension ABI { kind = .testCaseEnded case let .testCaseCancelled(skipInfo): kind = .testCaseCancelled + _comments = Array(skipInfo.comment).map(\.rawValue) _sourceLocation = skipInfo.sourceLocation case .testEnded: kind = .testEnded case let .testSkipped(skipInfo): kind = .testSkipped + _comments = Array(skipInfo.comment).map(\.rawValue) _sourceLocation = skipInfo.sourceLocation case let .testCancelled(skipInfo): kind = .testCancelled + _comments = Array(skipInfo.comment).map(\.rawValue) _sourceLocation = skipInfo.sourceLocation case .runEnded: kind = .runEnded diff --git a/Sources/Testing/ExitTests/ExitTest.swift b/Sources/Testing/ExitTests/ExitTest.swift index c716e421c..c82430eb5 100644 --- a/Sources/Testing/ExitTests/ExitTest.swift +++ b/Sources/Testing/ExitTests/ExitTest.swift @@ -291,9 +291,10 @@ extension ExitTest { current.pointee = self.unsafeCopy() } - do { + let error = await Issue.withErrorRecording(at: nil) { try await body(&self) - } catch { + } + if let error { _errorInMain(error) } @@ -768,7 +769,7 @@ extension ExitTest { } configuration.eventHandler = { event, eventContext in switch event.kind { - case .issueRecorded, .valueAttached: + case .issueRecorded, .valueAttached, .testCancelled, .testCaseCancelled: eventHandler(event, eventContext) default: // Don't forward other kinds of event. @@ -1042,11 +1043,15 @@ extension ExitTest { return } + lazy var comments: [Comment] = event._comments?.map(Comment.init(rawValue:)) ?? [] + lazy var sourceContext = SourceContext( + backtrace: nil, // A backtrace from the child process will have the wrong address space. + sourceLocation: event._sourceLocation + ) if let issue = event.issue { // Translate the issue back into a "real" issue and record it // in the parent process. This translation is, of course, lossy // due to the process boundary, but we make a best effort. - let comments: [Comment] = event.messages.map(\.text).map(Comment.init(rawValue:)) let issueKind: Issue.Kind = if let error = issue._error { .errorCaught(error) } else { @@ -1060,10 +1065,6 @@ extension ExitTest { // Prior to 6.3, all Issues are errors .error } - let sourceContext = SourceContext( - backtrace: nil, // `issue._backtrace` will have the wrong address space. - sourceLocation: issue.sourceLocation - ) var issueCopy = Issue(kind: issueKind, severity: severity, comments: comments, sourceContext: sourceContext) if issue.isKnown { // The known issue comment, if there was one, is already included in @@ -1073,6 +1074,10 @@ extension ExitTest { issueCopy.record() } else if let attachment = event.attachment { Attachment.record(attachment, sourceLocation: event._sourceLocation!) + } else if case .testCancelled = event.kind { + _ = try? Test.cancel(comments: comments, sourceContext: sourceContext) + } else if case .testCaseCancelled = event.kind { + _ = try? Test.Case.cancel(comments: comments, sourceContext: sourceContext) } } diff --git a/Sources/Testing/Issues/Issue+Recording.swift b/Sources/Testing/Issues/Issue+Recording.swift index a53d3e5c0..50cdbbc1f 100644 --- a/Sources/Testing/Issues/Issue+Recording.swift +++ b/Sources/Testing/Issues/Issue+Recording.swift @@ -159,7 +159,8 @@ extension Issue { /// allowing it to propagate to the caller. /// /// - Parameters: - /// - sourceLocation: The source location to attribute any caught error to. + /// - sourceLocation: The source location to attribute any caught error to, + /// if available. /// - configuration: The test configuration to use when recording an issue. /// The default value is ``Configuration/current``. /// - body: A closure that might throw an error. @@ -168,7 +169,7 @@ extension Issue { /// caught, otherwise `nil`. @discardableResult static func withErrorRecording( - at sourceLocation: SourceLocation, + at sourceLocation: SourceLocation?, configuration: Configuration? = nil, _ body: () throws -> Void ) -> (any Error)? { @@ -202,7 +203,8 @@ extension Issue { /// issue instead of allowing it to propagate to the caller. /// /// - Parameters: - /// - sourceLocation: The source location to attribute any caught error to. + /// - sourceLocation: The source location to attribute any caught error to, + /// if available. /// - configuration: The test configuration to use when recording an issue. /// The default value is ``Configuration/current``. /// - isolation: The actor to which `body` is isolated, if any. @@ -212,7 +214,7 @@ extension Issue { /// caught, otherwise `nil`. @discardableResult static func withErrorRecording( - at sourceLocation: SourceLocation, + at sourceLocation: SourceLocation?, configuration: Configuration? = nil, isolation: isolated (any Actor)? = #isolation, _ body: () async throws -> Void diff --git a/Sources/Testing/Test+Cancellation.swift b/Sources/Testing/Test+Cancellation.swift index 53b05c6b3..eb87f3620 100644 --- a/Sources/Testing/Test+Cancellation.swift +++ b/Sources/Testing/Test+Cancellation.swift @@ -16,7 +16,7 @@ protocol TestCancellable: Sendable { /// Cancel the current instance of this type. /// /// - Parameters: - /// - comment: A comment describing why you are cancelling the test. + /// - comments: Comments describing why you are cancelling the test/case. /// - sourceContext: The source context to which the testing library will /// attribute the cancellation. /// @@ -26,7 +26,7 @@ protocol TestCancellable: Sendable { /// Note that the public ``Test/cancel(_:sourceLocation:)`` function has a /// different signature and accepts a source location rather than a source /// context value. - static func cancel(comment: Comment?, sourceContext: @autoclosure () -> SourceContext) throws -> Never + static func cancel(comments: [Comment], sourceContext: @autoclosure () -> SourceContext) throws -> Never /// Make an instance of ``Event/Kind`` appropriate for an instance of this /// type. @@ -96,7 +96,7 @@ extension TestCancellable { // The current task was cancelled, so cancel the test case or test // associated with it. _ = try? Self.cancel( - comment: nil, + comments: [], sourceContext: SourceContext(backtrace: .current(), sourceLocation: nil) ) } @@ -112,13 +112,13 @@ extension TestCancellable { /// - cancellableValue: The test or test case to cancel, or `nil` if neither /// is set and we need fallback handling. /// - testAndTestCase: The test and test case to use when posting an event. -/// - comment: A comment describing why you are cancelling the test/case. +/// - comments: Comments describing why you are cancelling the test/case. /// - sourceContext: The source context to which the testing library will /// attribute the cancellation. /// /// - Throws: An instance of ``SkipInfo`` describing the cancellation. -private func _cancel(_ cancellableValue: T?, for testAndTestCase: (Test?, Test.Case?), comment: Comment?, sourceContext: @autoclosure () -> SourceContext) throws -> Never where T: TestCancellable { - var skipInfo = SkipInfo(comment: comment, sourceContext: .init(backtrace: nil, sourceLocation: nil)) +private func _cancel(_ cancellableValue: T?, for testAndTestCase: (Test?, Test.Case?), comments: [Comment], sourceContext: @autoclosure () -> SourceContext) throws -> Never where T: TestCancellable { + var skipInfo = SkipInfo(comment: comments.first, sourceContext: .init(backtrace: nil, sourceLocation: nil)) if cancellableValue != nil { // If the current test case is still running, cancel its task and clear its @@ -133,24 +133,25 @@ private func _cancel(_ cancellableValue: T?, for testAndTestCase: (Test?, Tes Event.post(T.makeCancelledEventKind(with: skipInfo), for: testAndTestCase) } } else { - // The current task isn't associated with a test case, so just cancel it - // and (try to) record an API misuse issue. + // The current task isn't associated with a test/case, so just cancel the + // task. withUnsafeCurrentTask { task in task?.cancel() } - var comments: [Comment] = if ExitTest.current != nil { - // Attempted to cancel the test or test case from within an exit test. The - // semantics of such an action aren't yet well-defined. - ["Attempted to cancel the current test or test case from within an exit test."] + if ExitTest.current != nil { + // This code is running in an exit test. We don't have a "current test" or + // "current test case" in the child process, so we'll let the parent + // process sort that out. + skipInfo.sourceContext = sourceContext() + Event.post(T.makeCancelledEventKind(with: skipInfo), for: (nil, nil)) } else { - ["Attempted to cancel the current test or test case, but one is not associated with the current task."] - } - if let comment { - comments.append(comment) + // Record an API misuse issue for trying to cancel the current test/case + // outside of any useful context. + let comments = ["Attempted to cancel the current test or test case, but one is not associated with the current task."] + comments + let issue = Issue(kind: .apiMisused, comments: comments, sourceContext: sourceContext()) + issue.record() } - let issue = Issue(kind: .apiMisused, comments: comments, sourceContext: sourceContext()) - issue.record() } throw skipInfo @@ -188,7 +189,11 @@ extension Test: TestCancellable { /// test or if it has already finished running, this function throws an error /// but does not attempt to cancel the test a second time. /// - /// - Note: You cannot cancel a test from within the body of an [exit test](doc:exit-testing). + /// @Comment { + /// TODO: Document the interaction between an exit test and test + /// cancellation. In particular, the error thrown by this function isn't + /// thrown into the parent process. + /// } /// /// To cancel the current test case but leave other test cases of the current /// test alone, call ``Test/Case/cancel(_:sourceLocation:)`` instead. @@ -199,14 +204,14 @@ extension Test: TestCancellable { @_spi(Experimental) public static func cancel(_ comment: Comment? = nil, sourceLocation: SourceLocation = #_sourceLocation) throws -> Never { try Self.cancel( - comment: comment, + comments: Array(comment), sourceContext: SourceContext(backtrace: .current(), sourceLocation: sourceLocation) ) } - static func cancel(comment: Comment?, sourceContext: @autoclosure () -> SourceContext) throws -> Never { + static func cancel(comments: [Comment], sourceContext: @autoclosure () -> SourceContext) throws -> Never { let test = Test.current - try _cancel(test, for: (test, nil), comment: comment, sourceContext: sourceContext()) + try _cancel(test, for: (test, nil), comments: comments, sourceContext: sourceContext()) } static func makeCancelledEventKind(with skipInfo: SkipInfo) -> Event.Kind { @@ -246,7 +251,11 @@ extension Test.Case: TestCancellable { /// function throws an error but does not attempt to cancel the test case a /// second time. /// - /// - Note: You cannot cancel a test case from within the body of an [exit test](doc:exit-testing). + /// @Comment { + /// TODO: Document the interaction between an exit test and test + /// cancellation. In particular, the error thrown by this function isn't + /// thrown into the parent process. + /// } /// /// To cancel all test cases in the current test, call /// ``Test/cancel(_:sourceLocation:)`` instead. @@ -257,22 +266,22 @@ extension Test.Case: TestCancellable { @_spi(Experimental) public static func cancel(_ comment: Comment? = nil, sourceLocation: SourceLocation = #_sourceLocation) throws -> Never { try Self.cancel( - comment: comment, + comments: Array(comment), sourceContext: SourceContext(backtrace: .current(), sourceLocation: sourceLocation) ) } - static func cancel(comment: Comment?, sourceContext: @autoclosure () -> SourceContext) throws -> Never { + static func cancel(comments: [Comment], sourceContext: @autoclosure () -> SourceContext) throws -> Never { let test = Test.current let testCase = Test.Case.current let sourceContext = sourceContext() // evaluated twice, avoid laziness do { // Cancel the current test case (if it's nil, that's the API misuse path.) - try _cancel(testCase, for: (test, testCase), comment: comment, sourceContext: sourceContext) + try _cancel(testCase, for: (test, testCase), comments: comments, sourceContext: sourceContext) } catch _ where test?.isParameterized == false { // The current test is not parameterized, so cancel the whole test too. - try _cancel(test, for: (test, nil), comment: comment, sourceContext: sourceContext) + try _cancel(test, for: (test, nil), comments: comments, sourceContext: sourceContext) } } diff --git a/Tests/TestingTests/TestCancellationTests.swift b/Tests/TestingTests/TestCancellationTests.swift index 5a09b8e32..f568edb03 100644 --- a/Tests/TestingTests/TestCancellationTests.swift +++ b/Tests/TestingTests/TestCancellationTests.swift @@ -126,12 +126,26 @@ } #if !SWT_NO_EXIT_TESTS - @Test func `Cancelling from within an exit test records an issue`() async { - await testCancellation(issueRecorded: 1) { configuration in + @Test func `Cancelling the current test from within an exit test`() async { + await testCancellation(testCancelled: 1, testCaseCancelled: 1) { configuration in + await Test { + await #expect(processExitsWith: .success) { + try Test.cancel("Cancelled test") + } + #expect(Task.isCancelled) + try Task.checkCancellation() + }.run(configuration: configuration) + } + } + + @Test func `Cancelling the current test case from within an exit test`() async { + await testCancellation(testCancelled: 1, testCaseCancelled: 1) { configuration in await Test { await #expect(processExitsWith: .success) { - _ = try? Test.cancel("Cancelled test") + try Test.Case.cancel("Cancelled test") } + #expect(Task.isCancelled) + try Task.checkCancellation() }.run(configuration: configuration) } } From 69feaccb5ac3a365e2db59209303bdce36c646d1 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Tue, 26 Aug 2025 13:29:19 -0400 Subject: [PATCH 19/24] Documentation tweaks and testing Task.cancel() in an exit test --- Sources/Testing/Test+Cancellation.swift | 18 ++++++++++-------- Tests/TestingTests/TestCancellationTests.swift | 12 ++++++++++++ 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/Sources/Testing/Test+Cancellation.swift b/Sources/Testing/Test+Cancellation.swift index eb87f3620..b1dfce93f 100644 --- a/Sources/Testing/Test+Cancellation.swift +++ b/Sources/Testing/Test+Cancellation.swift @@ -192,15 +192,16 @@ extension Test: TestCancellable { /// @Comment { /// TODO: Document the interaction between an exit test and test /// cancellation. In particular, the error thrown by this function isn't - /// thrown into the parent process. + /// thrown into the parent process and task cancellation doesn't propagate + /// (because the exit test _de facto_ runs in a detached task.) /// } /// - /// To cancel the current test case but leave other test cases of the current - /// test alone, call ``Test/Case/cancel(_:sourceLocation:)`` instead. - /// /// - Important: If the current task is not associated with a test (for /// example, because it was created with [`Task.detached(name:priority:operation:)`](https://developer.apple.com/documentation/swift/task/detached(name:priority:operation:)-795w1)) /// this function records an issue and cancels the current task. + /// + /// To cancel the current test case but leave other test cases of the current + /// test alone, call ``Test/Case/cancel(_:sourceLocation:)`` instead. @_spi(Experimental) public static func cancel(_ comment: Comment? = nil, sourceLocation: SourceLocation = #_sourceLocation) throws -> Never { try Self.cancel( @@ -254,15 +255,16 @@ extension Test.Case: TestCancellable { /// @Comment { /// TODO: Document the interaction between an exit test and test /// cancellation. In particular, the error thrown by this function isn't - /// thrown into the parent process. + /// thrown into the parent process and task cancellation doesn't propagate + /// (because the exit test _de facto_ runs in a detached task.) /// } /// - /// To cancel all test cases in the current test, call - /// ``Test/cancel(_:sourceLocation:)`` instead. - /// /// - Important: If the current task is not associated with a test case (for /// example, because it was created with [`Task.detached(name:priority:operation:)`](https://developer.apple.com/documentation/swift/task/detached(name:priority:operation:)-795w1)) /// this function records an issue and cancels the current task. + /// + /// To cancel all test cases in the current test, call + /// ``Test/cancel(_:sourceLocation:)`` instead. @_spi(Experimental) public static func cancel(_ comment: Comment? = nil, sourceLocation: SourceLocation = #_sourceLocation) throws -> Never { try Self.cancel( diff --git a/Tests/TestingTests/TestCancellationTests.swift b/Tests/TestingTests/TestCancellationTests.swift index f568edb03..57a6c4e63 100644 --- a/Tests/TestingTests/TestCancellationTests.swift +++ b/Tests/TestingTests/TestCancellationTests.swift @@ -149,6 +149,18 @@ }.run(configuration: configuration) } } + + @Test func `Cancelling the current task in an exit test doesn't cancel the test`() async { + await testCancellation(testCancelled: 0, testCaseCancelled: 0) { configuration in + await Test { + await #expect(processExitsWith: .success) { + withUnsafeCurrentTask { $0?.cancel() } + } + #expect(!Task.isCancelled) + try Task.checkCancellation() + }.run(configuration: configuration) + } + } #endif } From c2d0090632b6d219d6aff2e272dc5bc28b740cdb Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Tue, 26 Aug 2025 18:38:00 -0400 Subject: [PATCH 20/24] Make sure CancellationError is only treated as a skip during trait evaluation if the task was cancelled --- Sources/Testing/Running/Runner.Plan.swift | 2 +- Tests/TestingTests/TestCancellationTests.swift | 13 ++++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/Sources/Testing/Running/Runner.Plan.swift b/Sources/Testing/Running/Runner.Plan.swift index 93af2deab..b7edacfe2 100644 --- a/Sources/Testing/Running/Runner.Plan.swift +++ b/Sources/Testing/Running/Runner.Plan.swift @@ -219,7 +219,7 @@ extension Runner.Plan { } catch let error as SkipInfo { action = .skip(error) break - } catch is CancellationError { + } catch is CancellationError where Task.isCancelled { // Synthesize skip info for this cancellation error. let sourceContext = SourceContext(backtrace: .current(), sourceLocation: nil) let skipInfo = SkipInfo(comment: nil, sourceContext: sourceContext) diff --git a/Tests/TestingTests/TestCancellationTests.swift b/Tests/TestingTests/TestCancellationTests.swift index 57a6c4e63..3d7fb819e 100644 --- a/Tests/TestingTests/TestCancellationTests.swift +++ b/Tests/TestingTests/TestCancellationTests.swift @@ -109,6 +109,13 @@ } } + @Test func `Throwing CancellationError while evaluating traits without cancelling the test task`() async { + await testCancellation(issueRecorded: 1) { configuration in + await Test(CancelledTrait(throwsWithoutCancelling: true)) { + }.run(configuration: configuration) + } + } + @Test func `Cancelling a test while evaluating traits skips the test`() async { await testCancellation(testSkipped: 1) { configuration in await Test(CancelledTrait()) { @@ -197,12 +204,16 @@ final class TestCancellationTests: XCTestCase { // MARK: - Fixtures struct CancelledTrait: TestTrait { + var throwsWithoutCancelling = false var cancelsTask = false func prepare(for test: Test) async throws { + if throwsWithoutCancelling { + throw CancellationError() + } if cancelsTask { withUnsafeCurrentTask { $0?.cancel() } - throw CancellationError() + try Task.checkCancellation() } try Test.cancel("Cancelled from trait") } From e2f518b10eb26410815d3e4db2e0019b31e3bca3 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Wed, 27 Aug 2025 12:40:35 -0400 Subject: [PATCH 21/24] Remove some trys --- Sources/Testing/Running/Runner.swift | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/Sources/Testing/Running/Runner.swift b/Sources/Testing/Running/Runner.swift index 14de2a15b..d5a844fba 100644 --- a/Sources/Testing/Running/Runner.swift +++ b/Sources/Testing/Running/Runner.swift @@ -155,7 +155,7 @@ extension Runner { private static func _forEach( in sequence: some Sequence, _ body: @Sendable @escaping (E) async throws -> Void - ) async throws where E: Sendable { + ) async rethrows where E: Sendable { try await withThrowingTaskGroup { taskGroup in for element in sequence { // Each element gets its own subtask to run in. @@ -246,7 +246,7 @@ extension Runner { try await _applyScopingTraits(for: step.test, testCase: nil) { // Run the test function at this step (if one is present.) if let testCases = step.test.testCases { - try await _runTestCases(testCases, within: step) + await _runTestCases(testCases, within: step) } // Run the children of this test (i.e. the tests in this suite.) @@ -326,20 +326,17 @@ extension Runner { /// - testCases: The test cases to be run. /// - step: The runner plan step associated with this test case. /// - /// - Throws: Whatever is thrown from a test case's body. Thrown errors are - /// normally reported as test failures. - /// /// If parallelization is supported and enabled, the generated test cases will /// be run in parallel using a task group. - private static func _runTestCases(_ testCases: some Sequence, within step: Plan.Step) async throws { + private static func _runTestCases(_ testCases: some Sequence, within step: Plan.Step) async { // Apply the configuration's test case filter. let testCaseFilter = _configuration.testCaseFilter let testCases = testCases.lazy.filter { testCase in testCaseFilter(testCase, step.test) } - try await _forEach(in: testCases) { testCase in - try await _runTestCase(testCase, within: step) + await _forEach(in: testCases) { testCase in + await _runTestCase(testCase, within: step) } } @@ -349,12 +346,9 @@ extension Runner { /// - testCase: The test case to run. /// - step: The runner plan step associated with this test case. /// - /// - Throws: Whatever is thrown from the test case's body. Thrown errors - /// are normally reported as test failures. - /// /// This function sets ``Test/Case/current``, then invokes the test case's /// body closure. - private static func _runTestCase(_ testCase: Test.Case, within step: Plan.Step) async throws { + private static func _runTestCase(_ testCase: Test.Case, within step: Plan.Step) async { let configuration = _configuration Event.post(.testCaseStarted, for: (step.test, testCase), configuration: configuration) From 9e7c1370d533d599d5074765cc07870800adf9eb Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Wed, 27 Aug 2025 13:12:25 -0400 Subject: [PATCH 22/24] Add comment explaining weird task group --- Sources/Testing/Running/Runner.Plan.swift | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Sources/Testing/Running/Runner.Plan.swift b/Sources/Testing/Running/Runner.Plan.swift index b7edacfe2..a1a17d51b 100644 --- a/Sources/Testing/Running/Runner.Plan.swift +++ b/Sources/Testing/Running/Runner.Plan.swift @@ -205,6 +205,11 @@ extension Runner.Plan { /// error that was thrown during trait evaluation. If more than one error /// was thrown, the first-caught error is returned. private static func _determineAction(for test: Test) async -> (Action, (any Error)?) { + // We use a task group here with a single child task so that, if the trait + // code calls Test.cancel() we don't end up cancelling the entire test run. + // We could also model this as an unstructured task except that they aren't + // available in the "task-to-thread" concurrency model. + // // FIXME: Parallelize this work. Calling `prepare(...)` on all traits and // evaluating all test arguments should be safely parallelizable. await withTaskGroup(returning: (Action, (any Error)?).self) { taskGroup in From 3982afbe4ae8ed1c09937c5c74b991dcca56abd5 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Wed, 27 Aug 2025 13:36:06 -0400 Subject: [PATCH 23/24] Add placeholder text to DocC bundle --- .../Testing.docc/EnablingAndDisabling.md | 17 ++++++++ .../Testing.docc/MigratingFromXCTest.md | 39 +++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/Sources/Testing/Testing.docc/EnablingAndDisabling.md b/Sources/Testing/Testing.docc/EnablingAndDisabling.md index 9fab8eeab..7e3f31bde 100644 --- a/Sources/Testing/Testing.docc/EnablingAndDisabling.md +++ b/Sources/Testing/Testing.docc/EnablingAndDisabling.md @@ -120,3 +120,20 @@ func allIngredientsAvailable(for food: Food) -> Bool { ... } ) func makeSundae() async throws { ... } ``` + + diff --git a/Sources/Testing/Testing.docc/MigratingFromXCTest.md b/Sources/Testing/Testing.docc/MigratingFromXCTest.md index 81434b8c3..fcf1f529d 100644 --- a/Sources/Testing/Testing.docc/MigratingFromXCTest.md +++ b/Sources/Testing/Testing.docc/MigratingFromXCTest.md @@ -556,6 +556,45 @@ test function with an instance of this trait type to control whether it runs: } } + + ### Annotate known issues A test may have a known issue that sometimes or always prevents it from passing. From 33283d74e3799a4320b5f8150eedd268d56065e2 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Wed, 27 Aug 2025 14:44:13 -0400 Subject: [PATCH 24/24] Add comments per @stmontgomery's feedback --- .../ABI/Encoded/ABI.EncodedEvent.swift | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/Sources/Testing/ABI/Encoded/ABI.EncodedEvent.swift b/Sources/Testing/ABI/Encoded/ABI.EncodedEvent.swift index b7a94c5f4..a78f86368 100644 --- a/Sources/Testing/ABI/Encoded/ABI.EncodedEvent.swift +++ b/Sources/Testing/ABI/Encoded/ABI.EncodedEvent.swift @@ -68,11 +68,32 @@ extension ABI { /// The comments the test author provided for this event, if any. /// + /// The value of this property contains the comments related to the primary + /// user action that caused this event to be generated. + /// + /// Some kinds of events have additional associated comments. For example, + /// when using ``withKnownIssue(_:isIntermittent:sourceLocation:_:)``, there + /// can be separate comments for the "underlying" issue versus the known + /// issue matcher, and either can be `nil`. In such cases, the secondary + /// comment(s) are represented via a distinct property depending on the kind + /// of that event. + /// /// - Warning: Comments at this level are not yet part of the JSON schema. var _comments: [String]? /// A source location associated with this event, if any. /// + /// The value of this property represents the source location most closely + /// related to the primary user action that caused this event to be + /// generated. + /// + /// Some kinds of events have additional associated source locations. For + /// example, when using ``withKnownIssue(_:isIntermittent:sourceLocation:_:)``, + /// there can be separate source locations for the "underlying" issue versus + /// the known issue matcher. In such cases, the secondary source location(s) + /// are represented via a distinct property depending on the kind of that + /// event. + /// /// - Warning: Source locations at this level of the JSON schema are not yet /// part of said JSON schema. var _sourceLocation: SourceLocation?