diff --git a/Sources/Testing/ABI/Encoded/ABI.EncodedAttachment.swift b/Sources/Testing/ABI/Encoded/ABI.EncodedAttachment.swift index 9275da1ed..a2400f9bf 100644 --- a/Sources/Testing/ABI/Encoded/ABI.EncodedAttachment.swift +++ b/Sources/Testing/ABI/Encoded/ABI.EncodedAttachment.swift @@ -38,12 +38,6 @@ extension ABI { /// - Warning: Inline attachment content is not yet part of the JSON schema. var _bytes: Bytes? - /// The source location where this attachment was created. - /// - /// - Warning: Attachment source locations are not yet part of the JSON - /// schema. - var _sourceLocation: SourceLocation? - init(encoding attachment: borrowing Attachment, in eventContext: borrowing Event.Context) { path = attachment.fileSystemPath @@ -55,8 +49,6 @@ extension ABI { return Bytes(rawValue: [UInt8](bytes)) } } - - _sourceLocation = attachment.sourceLocation } } diff --git a/Sources/Testing/ABI/Encoded/ABI.EncodedEvent.swift b/Sources/Testing/ABI/Encoded/ABI.EncodedEvent.swift index 73e7db2ac..a78f86368 100644 --- a/Sources/Testing/ABI/Encoded/ABI.EncodedEvent.swift +++ b/Sources/Testing/ABI/Encoded/ABI.EncodedEvent.swift @@ -29,8 +29,10 @@ extension ABI { case issueRecorded case valueAttached case testCaseEnded + case testCaseCancelled = "_testCaseCancelled" case testEnded case testSkipped + case testCancelled = "_testCancelled" case runEnded } @@ -64,6 +66,38 @@ extension ABI { /// - Warning: Test cases are not yet part of the JSON schema. var _testCase: EncodedTestCase? + /// The comments the test author provided for this event, if any. + /// + /// The value of this property contains the comments related to the primary + /// user action that caused this event to be generated. + /// + /// Some kinds of events have additional associated comments. For example, + /// when using ``withKnownIssue(_:isIntermittent:sourceLocation:_:)``, there + /// can be separate comments for the "underlying" issue versus the known + /// issue matcher, and either can be `nil`. In such cases, the secondary + /// comment(s) are represented via a distinct property depending on the kind + /// of that event. + /// + /// - Warning: Comments at this level are not yet part of the JSON schema. + var _comments: [String]? + + /// A source location associated with this event, if any. + /// + /// The value of this property represents the source location most closely + /// related to the primary user action that caused this event to be + /// generated. + /// + /// Some kinds of events have additional associated source locations. For + /// example, when using ``withKnownIssue(_:isIntermittent:sourceLocation:_:)``, + /// there can be separate source locations for the "underlying" issue versus + /// the known issue matcher. In such cases, the secondary source location(s) + /// are represented via a distinct property depending on the kind of that + /// event. + /// + /// - Warning: Source locations at this level of the JSON schema are not yet + /// part of said JSON schema. + var _sourceLocation: SourceLocation? + init?(encoding event: borrowing Event, in eventContext: borrowing Event.Context, messages: borrowing [Event.HumanReadableOutputRecorder.Message]) { switch event.kind { case .runStarted: @@ -78,18 +112,31 @@ extension ABI { case let .issueRecorded(recordedIssue): kind = .issueRecorded issue = EncodedIssue(encoding: recordedIssue, in: eventContext) + _comments = recordedIssue.comments.map(\.rawValue) + _sourceLocation = recordedIssue.sourceLocation case let .valueAttached(attachment): kind = .valueAttached self.attachment = EncodedAttachment(encoding: attachment, in: eventContext) + _sourceLocation = attachment.sourceLocation case .testCaseEnded: if eventContext.test?.isParameterized == false { return nil } kind = .testCaseEnded + case let .testCaseCancelled(skipInfo): + kind = .testCaseCancelled + _comments = Array(skipInfo.comment).map(\.rawValue) + _sourceLocation = skipInfo.sourceLocation case .testEnded: kind = .testEnded - case .testSkipped: + case let .testSkipped(skipInfo): kind = .testSkipped + _comments = Array(skipInfo.comment).map(\.rawValue) + _sourceLocation = skipInfo.sourceLocation + case let .testCancelled(skipInfo): + kind = .testCancelled + _comments = Array(skipInfo.comment).map(\.rawValue) + _sourceLocation = skipInfo.sourceLocation case .runEnded: kind = .runEnded default: diff --git a/Sources/Testing/CMakeLists.txt b/Sources/Testing/CMakeLists.txt index 531d27cfc..d2cd21ab9 100644 --- a/Sources/Testing/CMakeLists.txt +++ b/Sources/Testing/CMakeLists.txt @@ -93,6 +93,7 @@ add_library(Testing Test.ID.Selection.swift Test.ID.swift Test.swift + Test+Cancellation.swift Test+Discovery.swift Test+Discovery+Legacy.swift Test+Macro.swift diff --git a/Sources/Testing/Events/Event.swift b/Sources/Testing/Events/Event.swift index 0be14ae88..d8daa3e89 100644 --- a/Sources/Testing/Events/Event.swift +++ b/Sources/Testing/Events/Event.swift @@ -74,6 +74,18 @@ public struct Event: Sendable { /// that was passed to the event handler along with this event. case testCaseEnded + /// A test case was cancelled. + /// + /// - Parameters: + /// - skipInfo: A ``SkipInfo`` with details about the cancelled test case. + /// + /// This event is generated by a call to ``Test/Case/cancel(_:sourceLocation:)``. + /// + /// The test case that was cancelled is contained in the ``Event/Context`` + /// instance that was passed to the event handler along with this event. + @_spi(Experimental) + indirect case testCaseCancelled(_ skipInfo: SkipInfo) + /// An expectation was checked with `#expect()` or `#require()`. /// /// - Parameters: @@ -121,6 +133,18 @@ public struct Event: Sendable { /// available from this event's ``Event/testID`` property. indirect case testSkipped(_ skipInfo: SkipInfo) + /// A test was cancelled. + /// + /// - Parameters: + /// - skipInfo: A ``SkipInfo`` with details about the cancelled test. + /// + /// This event is generated by a call to ``Test/cancel(_:sourceLocation:)``. + /// + /// The test that was cancelled is contained in the ``Event/Context`` + /// instance that was passed to the event handler along with this event. + @_spi(Experimental) + indirect case testCancelled(_ skipInfo: SkipInfo) + /// A step in the runner plan ended. /// /// - Parameters: @@ -395,6 +419,18 @@ extension Event.Kind { /// A test case ended. case testCaseEnded + /// A test case was cancelled. + /// + /// - Parameters: + /// - skipInfo: A ``SkipInfo`` with details about the cancelled test case. + /// + /// This event is generated by a call to ``Test/Case/cancel(_:sourceLocation:)``. + /// + /// The test case that was cancelled is contained in the ``Event/Context`` + /// instance that was passed to the event handler along with this event. + @_spi(Experimental) + indirect case testCaseCancelled(_ skipInfo: SkipInfo) + /// An expectation was checked with `#expect()` or `#require()`. /// /// - Parameters: @@ -431,6 +467,18 @@ extension Event.Kind { /// - skipInfo: A ``SkipInfo`` containing details about this skipped test. indirect case testSkipped(_ skipInfo: SkipInfo) + /// A test was cancelled. + /// + /// - Parameters: + /// - skipInfo: A ``SkipInfo`` with details about the cancelled test. + /// + /// This event is generated by a call to ``Test/cancel(_:sourceLocation:)``. + /// + /// The test that was cancelled is contained in the ``Event/Context`` + /// instance that was passed to the event handler along with this event. + @_spi(Experimental) + indirect case testCancelled(_ skipInfo: SkipInfo) + /// A step in the runner plan ended. /// /// - Parameters: @@ -479,6 +527,8 @@ extension Event.Kind { self = .testCaseStarted case .testCaseEnded: self = .testCaseEnded + case let .testCaseCancelled(skipInfo): + self = .testCaseCancelled(skipInfo) case let .expectationChecked(expectation): let expectationSnapshot = Expectation.Snapshot(snapshotting: expectation) self = Snapshot.expectationChecked(expectationSnapshot) @@ -490,6 +540,8 @@ extension Event.Kind { self = .testEnded case let .testSkipped(skipInfo): self = .testSkipped(skipInfo) + case let .testCancelled(skipInfo): + self = .testCancelled(skipInfo) case .planStepEnded: self = .planStepEnded case let .iterationEnded(index): diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index 434487e27..7497ad4cd 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -73,6 +73,9 @@ extension Event { /// The number of known issues recorded for the test. var knownIssueCount = 0 + + /// Information about the cancellation of this test or test case. + var cancellationInfo: SkipInfo? } /// Data tracked on a per-test basis. @@ -251,6 +254,7 @@ extension Event.HumanReadableOutputRecorder { 0 } let test = eventContext.test + let testCase = eventContext.testCase let keyPath = eventContext.keyPath let testName = if let test { if let displayName = test.displayName { @@ -310,6 +314,9 @@ extension Event.HumanReadableOutputRecorder { case .testCaseStarted: context.testData[keyPath] = .init(startInstant: instant) + case let .testCancelled(skipInfo), let .testCaseCancelled(skipInfo): + context.testData[keyPath]?.cancellationInfo = skipInfo + default: // These events do not manipulate the context structure. break @@ -404,21 +411,29 @@ extension Event.HumanReadableOutputRecorder { } else { "" } - return if issues.errorIssueCount > 0 { - CollectionOfOne( - Message( - symbol: .fail, - stringValue: "\(_capitalizedTitle(for: test)) \(testName)\(testCasesCount) failed after \(duration)\(issues.description)." - ) - ) + _formattedComments(for: test) + var cancellationComment = "." + let (symbol, verbed): (Event.Symbol, String) + if issues.errorIssueCount > 0 { + (symbol, verbed) = (.fail, "failed") + } else if !test.isParameterized, let cancellationInfo = testData.cancellationInfo { + if let comment = cancellationInfo.comment { + cancellationComment = ": \"\(comment.rawValue)\"" + } + (symbol, verbed) = (.skip, "was cancelled") } else { - [ - Message( - symbol: .pass(knownIssueCount: issues.knownIssueCount), - stringValue: "\(_capitalizedTitle(for: test)) \(testName)\(testCasesCount) passed after \(duration)\(issues.description)." - ) - ] + (symbol, verbed) = (.pass(knownIssueCount: issues.knownIssueCount), "passed") + } + + var result = [ + Message( + symbol: symbol, + stringValue: "\(_capitalizedTitle(for: test)) \(testName)\(testCasesCount) \(verbed) after \(duration)\(issues.description)\(cancellationComment)" + ) + ] + if issues.errorIssueCount > 0 { + result += _formattedComments(for: test) } + return result case let .testSkipped(skipInfo): let test = test! @@ -443,7 +458,7 @@ extension Event.HumanReadableOutputRecorder { } else { 0 } - let labeledArguments = if let testCase = eventContext.testCase { + let labeledArguments = if let testCase { testCase.labeledArguments() } else { "" @@ -523,7 +538,7 @@ extension Event.HumanReadableOutputRecorder { return result case .testCaseStarted: - guard let testCase = eventContext.testCase, testCase.isParameterized, let arguments = testCase.arguments else { + guard let testCase, testCase.isParameterized, let arguments = testCase.arguments else { break } @@ -535,7 +550,7 @@ extension Event.HumanReadableOutputRecorder { ] case .testCaseEnded: - guard verbosity > 0, let testCase = eventContext.testCase, testCase.isParameterized, let arguments = testCase.arguments else { + guard verbosity > 0, let test, let testCase, testCase.isParameterized, let arguments = testCase.arguments else { break } @@ -544,18 +559,28 @@ extension Event.HumanReadableOutputRecorder { let issues = _issueCounts(in: testDataGraph) let duration = testData.startInstant.descriptionOfDuration(to: instant) - let message = if issues.errorIssueCount > 0 { - Message( - symbol: .fail, - stringValue: "Test case passing \(arguments.count.counting("argument")) \(testCase.labeledArguments(includingQualifiedTypeNames: verbosity > 0)) to \(testName) failed after \(duration)\(issues.description)." - ) + var cancellationComment = "." + let (symbol, verbed): (Event.Symbol, String) + if issues.errorIssueCount > 0 { + (symbol, verbed) = (.fail, "failed") + } else if !test.isParameterized, let cancellationInfo = testData.cancellationInfo { + if let comment = cancellationInfo.comment { + cancellationComment = ": \"\(comment.rawValue)\"" + } + (symbol, verbed) = (.skip, "was cancelled") } else { + (symbol, verbed) = (.pass(knownIssueCount: issues.knownIssueCount), "passed") + } + return [ Message( - symbol: .pass(knownIssueCount: issues.knownIssueCount), - stringValue: "Test case passing \(arguments.count.counting("argument")) \(testCase.labeledArguments(includingQualifiedTypeNames: verbosity > 0)) to \(testName) passed after \(duration)\(issues.description)." + symbol: symbol, + stringValue: "Test case passing \(arguments.count.counting("argument")) \(testCase.labeledArguments(includingQualifiedTypeNames: verbosity > 0)) to \(testName) \(verbed) after \(duration)\(issues.description)\(cancellationComment)" ) - } - return [message] + ] + + case .testCancelled, .testCaseCancelled: + // Handled in .testEnded and .testCaseEnded + break case let .iterationEnded(index): guard let iterationStartInstant = context.iterationStartInstant else { diff --git a/Sources/Testing/ExitTests/ExitTest.swift b/Sources/Testing/ExitTests/ExitTest.swift index 5eb006b03..c82430eb5 100644 --- a/Sources/Testing/ExitTests/ExitTest.swift +++ b/Sources/Testing/ExitTests/ExitTest.swift @@ -291,9 +291,10 @@ extension ExitTest { current.pointee = self.unsafeCopy() } - do { + let error = await Issue.withErrorRecording(at: nil) { try await body(&self) - } catch { + } + if let error { _errorInMain(error) } @@ -768,7 +769,7 @@ extension ExitTest { } configuration.eventHandler = { event, eventContext in switch event.kind { - case .issueRecorded, .valueAttached: + case .issueRecorded, .valueAttached, .testCancelled, .testCaseCancelled: eventHandler(event, eventContext) default: // Don't forward other kinds of event. @@ -1042,11 +1043,15 @@ extension ExitTest { return } + lazy var comments: [Comment] = event._comments?.map(Comment.init(rawValue:)) ?? [] + lazy var sourceContext = SourceContext( + backtrace: nil, // A backtrace from the child process will have the wrong address space. + sourceLocation: event._sourceLocation + ) if let issue = event.issue { // Translate the issue back into a "real" issue and record it // in the parent process. This translation is, of course, lossy // due to the process boundary, but we make a best effort. - let comments: [Comment] = event.messages.map(\.text).map(Comment.init(rawValue:)) let issueKind: Issue.Kind = if let error = issue._error { .errorCaught(error) } else { @@ -1060,10 +1065,6 @@ extension ExitTest { // Prior to 6.3, all Issues are errors .error } - let sourceContext = SourceContext( - backtrace: nil, // `issue._backtrace` will have the wrong address space. - sourceLocation: issue.sourceLocation - ) var issueCopy = Issue(kind: issueKind, severity: severity, comments: comments, sourceContext: sourceContext) if issue.isKnown { // The known issue comment, if there was one, is already included in @@ -1072,7 +1073,11 @@ extension ExitTest { } issueCopy.record() } else if let attachment = event.attachment { - Attachment.record(attachment, sourceLocation: attachment._sourceLocation!) + Attachment.record(attachment, sourceLocation: event._sourceLocation!) + } else if case .testCancelled = event.kind { + _ = try? Test.cancel(comments: comments, sourceContext: sourceContext) + } else if case .testCaseCancelled = event.kind { + _ = try? Test.Case.cancel(comments: comments, sourceContext: sourceContext) } } diff --git a/Sources/Testing/Issues/Issue+Recording.swift b/Sources/Testing/Issues/Issue+Recording.swift index d99323777..50cdbbc1f 100644 --- a/Sources/Testing/Issues/Issue+Recording.swift +++ b/Sources/Testing/Issues/Issue+Recording.swift @@ -159,7 +159,8 @@ extension Issue { /// allowing it to propagate to the caller. /// /// - Parameters: - /// - sourceLocation: The source location to attribute any caught error to. + /// - sourceLocation: The source location to attribute any caught error to, + /// if available. /// - configuration: The test configuration to use when recording an issue. /// The default value is ``Configuration/current``. /// - body: A closure that might throw an error. @@ -168,7 +169,7 @@ extension Issue { /// caught, otherwise `nil`. @discardableResult static func withErrorRecording( - at sourceLocation: SourceLocation, + at sourceLocation: SourceLocation?, configuration: Configuration? = nil, _ body: () throws -> Void ) -> (any Error)? { @@ -185,6 +186,10 @@ extension Issue { // This error is thrown by expectation checking functions to indicate a // condition evaluated to `false`. Those functions record their own issue, // so we don't need to record another one redundantly. + } catch is SkipInfo, + is CancellationError where Task.isCancelled { + // This error represents control flow rather than an issue, so we suppress + // it here. } catch { let issue = Issue(for: error, sourceLocation: sourceLocation) issue.record(configuration: configuration) @@ -198,7 +203,8 @@ extension Issue { /// issue instead of allowing it to propagate to the caller. /// /// - Parameters: - /// - sourceLocation: The source location to attribute any caught error to. + /// - sourceLocation: The source location to attribute any caught error to, + /// if available. /// - configuration: The test configuration to use when recording an issue. /// The default value is ``Configuration/current``. /// - isolation: The actor to which `body` is isolated, if any. @@ -208,7 +214,7 @@ extension Issue { /// caught, otherwise `nil`. @discardableResult static func withErrorRecording( - at sourceLocation: SourceLocation, + at sourceLocation: SourceLocation?, configuration: Configuration? = nil, isolation: isolated (any Actor)? = #isolation, _ body: () async throws -> Void @@ -226,6 +232,10 @@ extension Issue { // This error is thrown by expectation checking functions to indicate a // condition evaluated to `false`. Those functions record their own issue, // so we don't need to record another one redundantly. + } catch is SkipInfo, + is CancellationError where Task.isCancelled { + // This error represents control flow rather than an issue, so we suppress + // it here. } catch { let issue = Issue(for: error, sourceLocation: sourceLocation) issue.record(configuration: configuration) diff --git a/Sources/Testing/Running/Runner.Plan.swift b/Sources/Testing/Running/Runner.Plan.swift index c89fdecb5..a1a17d51b 100644 --- a/Sources/Testing/Running/Runner.Plan.swift +++ b/Sources/Testing/Running/Runner.Plan.swift @@ -193,6 +193,57 @@ extension Runner.Plan { synthesizeSuites(in: &graph, sourceLocation: &sourceLocation) } + /// The basic "run" action. + private static let _runAction = Action.run(options: .init()) + + /// Determine what action to perform for a given test by preparing its traits. + /// + /// - Parameters: + /// - test: The test whose action will be determined. + /// + /// - Returns: A tuple containing the action to take for `test` as well as any + /// error that was thrown during trait evaluation. If more than one error + /// was thrown, the first-caught error is returned. + private static func _determineAction(for test: Test) async -> (Action, (any Error)?) { + // We use a task group here with a single child task so that, if the trait + // code calls Test.cancel() we don't end up cancelling the entire test run. + // We could also model this as an unstructured task except that they aren't + // available in the "task-to-thread" concurrency model. + // + // FIXME: Parallelize this work. Calling `prepare(...)` on all traits and + // evaluating all test arguments should be safely parallelizable. + await withTaskGroup(returning: (Action, (any Error)?).self) { taskGroup in + taskGroup.addTask { + var action = _runAction + var firstCaughtError: (any Error)? + + await Test.withCurrent(test) { + for trait in test.traits { + do { + try await trait.prepare(for: test) + } catch let error as SkipInfo { + action = .skip(error) + break + } catch is CancellationError where Task.isCancelled { + // Synthesize skip info for this cancellation error. + let sourceContext = SourceContext(backtrace: .current(), sourceLocation: nil) + let skipInfo = SkipInfo(comment: nil, sourceContext: sourceContext) + action = .skip(skipInfo) + break + } catch { + // Only preserve the first caught error + firstCaughtError = firstCaughtError ?? error + } + } + } + + return (action, firstCaughtError) + } + + return await taskGroup.first { _ in true }! + } + } + /// Construct a graph of runner plan steps for the specified tests. /// /// - Parameters: @@ -211,7 +262,7 @@ extension Runner.Plan { // Convert the list of test into a graph of steps. The actions for these // steps will all be .run() *unless* an error was thrown while examining // them, in which case it will be .recordIssue(). - let runAction = Action.run(options: .init()) + let runAction = _runAction var testGraph = Graph() var actionGraph = Graph(value: runAction) for test in tests { @@ -251,9 +302,6 @@ extension Runner.Plan { _recursivelyApplyTraits(to: &testGraph) // For each test value, determine the appropriate action for it. - // - // FIXME: Parallelize this work. Calling `prepare(...)` on all traits and - // evaluating all test arguments should be safely parallelizable. testGraph = await testGraph.mapValues { keyPath, test in // Skip any nil test, which implies this node is just a placeholder and // not actual test content. @@ -269,17 +317,7 @@ extension Runner.Plan { // But if any throw another kind of error, keep track of the first error // but continue walking, because if any subsequent traits throw a // `SkipInfo`, the error should not be recorded. - for trait in test.traits { - do { - try await trait.prepare(for: test) - } catch let error as SkipInfo { - action = .skip(error) - break - } catch { - // Only preserve the first caught error - firstCaughtError = firstCaughtError ?? error - } - } + (action, firstCaughtError) = await _determineAction(for: test) // If no trait specified that the test should be skipped, but one did // throw an error, then the action is to record an issue for that error. diff --git a/Sources/Testing/Running/Runner.RuntimeState.swift b/Sources/Testing/Running/Runner.RuntimeState.swift index 9ae299412..e88cea60b 100644 --- a/Sources/Testing/Running/Runner.RuntimeState.swift +++ b/Sources/Testing/Running/Runner.RuntimeState.swift @@ -206,7 +206,9 @@ extension Test { static func withCurrent(_ test: Self, perform body: () async throws -> R) async rethrows -> R { var runtimeState = Runner.RuntimeState.current ?? .init() runtimeState.test = test - return try await Runner.RuntimeState.$current.withValue(runtimeState, operation: body) + return try await Runner.RuntimeState.$current.withValue(runtimeState) { + try await test.withCancellationHandling(body) + } } } @@ -239,7 +241,9 @@ extension Test.Case { static func withCurrent(_ testCase: Self, perform body: () async throws -> R) async rethrows -> R { var runtimeState = Runner.RuntimeState.current ?? .init() runtimeState.testCase = testCase - return try await Runner.RuntimeState.$current.withValue(runtimeState, operation: body) + return try await Runner.RuntimeState.$current.withValue(runtimeState) { + try await testCase.withCancellationHandling(body) + } } } diff --git a/Sources/Testing/Running/Runner.swift b/Sources/Testing/Running/Runner.swift index bd1167b8e..d5a844fba 100644 --- a/Sources/Testing/Running/Runner.swift +++ b/Sources/Testing/Running/Runner.swift @@ -155,11 +155,11 @@ extension Runner { private static func _forEach( in sequence: some Sequence, _ body: @Sendable @escaping (E) async throws -> Void - ) async throws where E: Sendable { + ) async rethrows where E: Sendable { try await withThrowingTaskGroup { taskGroup in for element in sequence { // Each element gets its own subtask to run in. - _ = taskGroup.addTaskUnlessCancelled { + taskGroup.addTask { try await body(element) } @@ -190,9 +190,6 @@ extension Runner { /// /// - ``Runner/run()`` private static func _runStep(atRootOf stepGraph: Graph) async throws { - // Exit early if the task has already been cancelled. - try Task.checkCancellation() - // Whether to send a `.testEnded` event at the end of running this step. // Some steps' actions may not require a final event to be sent — for // example, a skip event only sends `.testSkipped`. @@ -243,10 +240,13 @@ extension Runner { if let step = stepGraph.value, case .run = step.action { await Test.withCurrent(step.test) { _ = await Issue.withErrorRecording(at: step.test.sourceLocation, configuration: configuration) { + // Exit early if the task has already been cancelled. + try Task.checkCancellation() + try await _applyScopingTraits(for: step.test, testCase: nil) { // Run the test function at this step (if one is present.) if let testCases = step.test.testCases { - try await _runTestCases(testCases, within: step) + await _runTestCases(testCases, within: step) } // Run the children of this test (i.e. the tests in this suite.) @@ -326,20 +326,17 @@ extension Runner { /// - testCases: The test cases to be run. /// - step: The runner plan step associated with this test case. /// - /// - Throws: Whatever is thrown from a test case's body. Thrown errors are - /// normally reported as test failures. - /// /// If parallelization is supported and enabled, the generated test cases will /// be run in parallel using a task group. - private static func _runTestCases(_ testCases: some Sequence, within step: Plan.Step) async throws { + private static func _runTestCases(_ testCases: some Sequence, within step: Plan.Step) async { // Apply the configuration's test case filter. let testCaseFilter = _configuration.testCaseFilter let testCases = testCases.lazy.filter { testCase in testCaseFilter(testCase, step.test) } - try await _forEach(in: testCases) { testCase in - try await _runTestCase(testCase, within: step) + await _forEach(in: testCases) { testCase in + await _runTestCase(testCase, within: step) } } @@ -349,15 +346,9 @@ extension Runner { /// - testCase: The test case to run. /// - step: The runner plan step associated with this test case. /// - /// - Throws: Whatever is thrown from the test case's body. Thrown errors - /// are normally reported as test failures. - /// /// This function sets ``Test/Case/current``, then invokes the test case's /// body closure. - private static func _runTestCase(_ testCase: Test.Case, within step: Plan.Step) async throws { - // Exit early if the task has already been cancelled. - try Task.checkCancellation() - + private static func _runTestCase(_ testCase: Test.Case, within step: Plan.Step) async { let configuration = _configuration Event.post(.testCaseStarted, for: (step.test, testCase), configuration: configuration) @@ -368,6 +359,9 @@ extension Runner { await Test.Case.withCurrent(testCase) { let sourceLocation = step.test.sourceLocation await Issue.withErrorRecording(at: sourceLocation, configuration: configuration) { + // Exit early if the task has already been cancelled. + try Task.checkCancellation() + try await withTimeLimit(for: step.test, configuration: configuration) { try await _applyScopingTraits(for: step.test, testCase: testCase) { try await testCase.body() diff --git a/Sources/Testing/Test+Cancellation.swift b/Sources/Testing/Test+Cancellation.swift new file mode 100644 index 000000000..b1dfce93f --- /dev/null +++ b/Sources/Testing/Test+Cancellation.swift @@ -0,0 +1,293 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +/// A protocol describing cancellable tests and test cases. +/// +/// This protocol is used to abstract away the common implementation of test and +/// test case cancellation. +protocol TestCancellable: Sendable { + /// Cancel the current instance of this type. + /// + /// - Parameters: + /// - comments: Comments describing why you are cancelling the test/case. + /// - sourceContext: The source context to which the testing library will + /// attribute the cancellation. + /// + /// - Throws: An error indicating that the current instance of this type has + /// been cancelled. + /// + /// Note that the public ``Test/cancel(_:sourceLocation:)`` function has a + /// different signature and accepts a source location rather than a source + /// context value. + static func cancel(comments: [Comment], sourceContext: @autoclosure () -> SourceContext) throws -> Never + + /// Make an instance of ``Event/Kind`` appropriate for an instance of this + /// type. + /// + /// - Parameters: + /// - skipInfo: The ``SkipInfo`` structure describing the cancellation. + /// + /// - Returns: An instance of ``Event/Kind`` that describes the cancellation. + static func makeCancelledEventKind(with skipInfo: SkipInfo) -> Event.Kind +} + +// MARK: - Tracking the current task + +/// A structure describing a reference to a task that is associated with some +/// ``TestCancellable`` value. +private struct _TaskReference: Sendable { + /// The unsafe underlying reference to the associated task. + private nonisolated(unsafe) var _unsafeCurrentTask = Locked() + + init() { + let unsafeCurrentTask = withUnsafeCurrentTask { $0 } + _unsafeCurrentTask = Locked(rawValue: unsafeCurrentTask) + } + + /// Take this instance's reference to its associated task. + /// + /// - Returns: An `UnsafeCurrentTask` instance, or `nil` if it was already + /// taken or if it was never available. + /// + /// This function consumes the reference to the task. After the first call, + /// subsequent calls on the same instance return `nil`. + func takeUnsafeCurrentTask() -> UnsafeCurrentTask? { + _unsafeCurrentTask.withLock { unsafeCurrentTask in + let result = unsafeCurrentTask + unsafeCurrentTask = nil + return result + } + } +} + +/// A dictionary of tracked tasks, keyed by types that conform to +/// ``TestCancellable``. +@TaskLocal +private var _currentTaskReferences = [ObjectIdentifier: _TaskReference]() + +extension TestCancellable { + /// Call a function while the ``unsafeCurrentTask`` property of this instance + /// is set to the current task. + /// + /// - Parameters: + /// - body: The function to invoke. + /// + /// - Returns: Whatever is returned by `body`. + /// + /// - Throws: Whatever is thrown by `body`. + /// + /// This function sets up a task cancellation handler and calls `body`. If + /// the current task, test, or test case is cancelled, it records a + /// corresponding cancellation event. + func withCancellationHandling(_ body: () async throws -> R) async rethrows -> R { + var currentTaskReferences = _currentTaskReferences + currentTaskReferences[ObjectIdentifier(Self.self)] = _TaskReference() + return try await $_currentTaskReferences.withValue(currentTaskReferences) { + try await withTaskCancellationHandler { + try await body() + } onCancel: { + // The current task was cancelled, so cancel the test case or test + // associated with it. + _ = try? Self.cancel( + comments: [], + sourceContext: SourceContext(backtrace: .current(), sourceLocation: nil) + ) + } + } + } +} + +// MARK: - + +/// The common implementation of cancellation for ``Test`` and ``Test/Case``. +/// +/// - Parameters: +/// - cancellableValue: The test or test case to cancel, or `nil` if neither +/// is set and we need fallback handling. +/// - testAndTestCase: The test and test case to use when posting an event. +/// - comments: Comments describing why you are cancelling the test/case. +/// - sourceContext: The source context to which the testing library will +/// attribute the cancellation. +/// +/// - Throws: An instance of ``SkipInfo`` describing the cancellation. +private func _cancel(_ cancellableValue: T?, for testAndTestCase: (Test?, Test.Case?), comments: [Comment], sourceContext: @autoclosure () -> SourceContext) throws -> Never where T: TestCancellable { + var skipInfo = SkipInfo(comment: comments.first, sourceContext: .init(backtrace: nil, sourceLocation: nil)) + + if cancellableValue != nil { + // If the current test case is still running, cancel its task and clear its + // task property (which signals that it has been cancelled.) + let task = _currentTaskReferences[ObjectIdentifier(T.self)]?.takeUnsafeCurrentTask() + task?.cancel() + + // If we just cancelled the current test case's task, post a corresponding + // event with the relevant skip info. + if task != nil { + skipInfo.sourceContext = sourceContext() + Event.post(T.makeCancelledEventKind(with: skipInfo), for: testAndTestCase) + } + } else { + // The current task isn't associated with a test/case, so just cancel the + // task. + withUnsafeCurrentTask { task in + task?.cancel() + } + + if ExitTest.current != nil { + // This code is running in an exit test. We don't have a "current test" or + // "current test case" in the child process, so we'll let the parent + // process sort that out. + skipInfo.sourceContext = sourceContext() + Event.post(T.makeCancelledEventKind(with: skipInfo), for: (nil, nil)) + } else { + // Record an API misuse issue for trying to cancel the current test/case + // outside of any useful context. + let comments = ["Attempted to cancel the current test or test case, but one is not associated with the current task."] + comments + let issue = Issue(kind: .apiMisused, comments: comments, sourceContext: sourceContext()) + issue.record() + } + } + + throw skipInfo +} + +// MARK: - Test cancellation + +extension Test: TestCancellable { + /// Cancel the current test. + /// + /// - Parameters: + /// - comment: A comment describing why you are cancelling the test. + /// - sourceLocation: The source location to which the testing library will + /// attribute the cancellation. + /// + /// - Throws: An error indicating that the current test case has been + /// cancelled. + /// + /// The testing library runs each test in its own task. When you call this + /// function, the testing library cancels the task associated with the current + /// test: + /// + /// ```swift + /// @Test func `Food truck is well-stocked`() throws { + /// guard businessHours.contains(.now) else { + /// try Test.cancel("We're off the clock.") + /// } + /// // ... + /// } + /// ``` + /// + /// If the current test is parameterized, all of its pending and running test + /// cases are cancelled. If the current test is a suite, all of its pending + /// and running tests are cancelled. If you have already cancelled the current + /// test or if it has already finished running, this function throws an error + /// but does not attempt to cancel the test a second time. + /// + /// @Comment { + /// TODO: Document the interaction between an exit test and test + /// cancellation. In particular, the error thrown by this function isn't + /// thrown into the parent process and task cancellation doesn't propagate + /// (because the exit test _de facto_ runs in a detached task.) + /// } + /// + /// - Important: If the current task is not associated with a test (for + /// example, because it was created with [`Task.detached(name:priority:operation:)`](https://developer.apple.com/documentation/swift/task/detached(name:priority:operation:)-795w1)) + /// this function records an issue and cancels the current task. + /// + /// To cancel the current test case but leave other test cases of the current + /// test alone, call ``Test/Case/cancel(_:sourceLocation:)`` instead. + @_spi(Experimental) + public static func cancel(_ comment: Comment? = nil, sourceLocation: SourceLocation = #_sourceLocation) throws -> Never { + try Self.cancel( + comments: Array(comment), + sourceContext: SourceContext(backtrace: .current(), sourceLocation: sourceLocation) + ) + } + + static func cancel(comments: [Comment], sourceContext: @autoclosure () -> SourceContext) throws -> Never { + let test = Test.current + try _cancel(test, for: (test, nil), comments: comments, sourceContext: sourceContext()) + } + + static func makeCancelledEventKind(with skipInfo: SkipInfo) -> Event.Kind { + .testCancelled(skipInfo) + } +} + +// MARK: - Test case cancellation + +extension Test.Case: TestCancellable { + /// Cancel the current test case. + /// + /// - Parameters: + /// - comment: A comment describing why you are cancelling the test case. + /// - sourceLocation: The source location to which the testing library will + /// attribute the cancellation. + /// + /// - Throws: An error indicating that the current test case has been + /// cancelled. + /// + /// The testing library runs each test case of a test in its own task. When + /// you call this function, the testing library cancels the task associated + /// with the current test case: + /// + /// ```swift + /// @Test(arguments: [Food.burger, .fries, .iceCream]) + /// func `Food truck is well-stocked`(_ food: Food) throws { + /// if food == .iceCream && Season.current == .winter { + /// try Test.Case.cancel("It's too cold for ice cream.") + /// } + /// // ... + /// } + /// ``` + /// + /// If the current test is parameterized, the test's other test cases continue + /// running. If the current test case has already been cancelled, this + /// function throws an error but does not attempt to cancel the test case a + /// second time. + /// + /// @Comment { + /// TODO: Document the interaction between an exit test and test + /// cancellation. In particular, the error thrown by this function isn't + /// thrown into the parent process and task cancellation doesn't propagate + /// (because the exit test _de facto_ runs in a detached task.) + /// } + /// + /// - Important: If the current task is not associated with a test case (for + /// example, because it was created with [`Task.detached(name:priority:operation:)`](https://developer.apple.com/documentation/swift/task/detached(name:priority:operation:)-795w1)) + /// this function records an issue and cancels the current task. + /// + /// To cancel all test cases in the current test, call + /// ``Test/cancel(_:sourceLocation:)`` instead. + @_spi(Experimental) + public static func cancel(_ comment: Comment? = nil, sourceLocation: SourceLocation = #_sourceLocation) throws -> Never { + try Self.cancel( + comments: Array(comment), + sourceContext: SourceContext(backtrace: .current(), sourceLocation: sourceLocation) + ) + } + + static func cancel(comments: [Comment], sourceContext: @autoclosure () -> SourceContext) throws -> Never { + let test = Test.current + let testCase = Test.Case.current + let sourceContext = sourceContext() // evaluated twice, avoid laziness + + do { + // Cancel the current test case (if it's nil, that's the API misuse path.) + try _cancel(testCase, for: (test, testCase), comments: comments, sourceContext: sourceContext) + } catch _ where test?.isParameterized == false { + // The current test is not parameterized, so cancel the whole test too. + try _cancel(test, for: (test, nil), comments: comments, sourceContext: sourceContext) + } + } + + static func makeCancelledEventKind(with skipInfo: SkipInfo) -> Event.Kind { + .testCaseCancelled(skipInfo) + } +} diff --git a/Sources/Testing/Testing.docc/EnablingAndDisabling.md b/Sources/Testing/Testing.docc/EnablingAndDisabling.md index 9fab8eeab..7e3f31bde 100644 --- a/Sources/Testing/Testing.docc/EnablingAndDisabling.md +++ b/Sources/Testing/Testing.docc/EnablingAndDisabling.md @@ -120,3 +120,20 @@ func allIngredientsAvailable(for food: Food) -> Bool { ... } ) func makeSundae() async throws { ... } ``` + + diff --git a/Sources/Testing/Testing.docc/MigratingFromXCTest.md b/Sources/Testing/Testing.docc/MigratingFromXCTest.md index 81434b8c3..fcf1f529d 100644 --- a/Sources/Testing/Testing.docc/MigratingFromXCTest.md +++ b/Sources/Testing/Testing.docc/MigratingFromXCTest.md @@ -556,6 +556,45 @@ test function with an instance of this trait type to control whether it runs: } } + + ### Annotate known issues A test may have a known issue that sometimes or always prevents it from passing. diff --git a/Tests/TestingTests/TestCancellationTests.swift b/Tests/TestingTests/TestCancellationTests.swift new file mode 100644 index 000000000..3d7fb819e --- /dev/null +++ b/Tests/TestingTests/TestCancellationTests.swift @@ -0,0 +1,228 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +@testable @_spi(Experimental) @_spi(ForToolsIntegrationOnly) import Testing + +@Suite(.serialized) struct `Test cancellation tests` { + func testCancellation(testCancelled: Int = 0, testSkipped: Int = 0, testCaseCancelled: Int = 0, issueRecorded: Int = 0, _ body: @Sendable (Configuration) async -> Void) async { + await confirmation("Test cancelled", expectedCount: testCancelled) { testCancelled in + await confirmation("Test skipped", expectedCount: testSkipped) { testSkipped in + await confirmation("Test case cancelled", expectedCount: testCaseCancelled) { testCaseCancelled in + await confirmation("Issue recorded", expectedCount: issueRecorded) { [issueRecordedCount = issueRecorded] issueRecorded in + var configuration = Configuration() + configuration.eventHandler = { event, _ in + switch event.kind { + case .testCancelled: + testCancelled() + case .testSkipped: + testSkipped() + case .testCaseCancelled: + testCaseCancelled() + case let .issueRecorded(issue): + if issueRecordedCount == 0 { + issue.record() + } + issueRecorded() + default: + break + } + } +#if !SWT_NO_EXIT_TESTS + configuration.exitTestHandler = ExitTest.handlerForEntryPoint() +#endif + await body(configuration) + } + } + } + } + } + + @Test func `Cancelling a test`() async { + await testCancellation(testCancelled: 1, testCaseCancelled: 1) { configuration in + await Test { + try Test.cancel("Cancelled test") + }.run(configuration: configuration) + } + } + + @Test func `Cancelling a non-parameterized test via Test.Case.cancel()`() async { + await testCancellation(testCancelled: 1, testCaseCancelled: 1) { configuration in + await Test { + try Test.Case.cancel("Cancelled test") + }.run(configuration: configuration) + } + } + + @Test func `Cancelling a test case in a parameterized test`() async { + await testCancellation(testCaseCancelled: 5, issueRecorded: 5) { configuration in + await Test(arguments: 0 ..< 10) { i in + if (i % 2) == 0 { + try Test.Case.cancel("\(i) is even!") + } + Issue.record("\(i) records an issue!") + }.run(configuration: configuration) + } + } + + @Test func `Cancelling an entire parameterized test`() async { + await testCancellation(testCancelled: 1, testCaseCancelled: 10) { configuration in + // .serialized to ensure that none of the cases complete before the first + // one cancels the test. + await Test(.serialized, arguments: 0 ..< 10) { i in + if i == 0 { + try Test.cancel("\(i) cancelled the test") + } + Issue.record("\(i) records an issue!") + }.run(configuration: configuration) + } + } + + @Test func `Cancelling a test by cancelling its task (throwing)`() async { + await testCancellation(testCancelled: 1, testCaseCancelled: 1) { configuration in + await Test { + withUnsafeCurrentTask { $0?.cancel() } + try Task.checkCancellation() + }.run(configuration: configuration) + } + } + + @Test func `Cancelling a test by cancelling its task (returning)`() async { + await testCancellation(testCancelled: 1, testCaseCancelled: 1) { configuration in + await Test { + withUnsafeCurrentTask { $0?.cancel() } + }.run(configuration: configuration) + } + } + + @Test func `Throwing CancellationError without cancelling the test task`() async { + await testCancellation(issueRecorded: 1) { configuration in + await Test { + throw CancellationError() + }.run(configuration: configuration) + } + } + + @Test func `Throwing CancellationError while evaluating traits without cancelling the test task`() async { + await testCancellation(issueRecorded: 1) { configuration in + await Test(CancelledTrait(throwsWithoutCancelling: true)) { + }.run(configuration: configuration) + } + } + + @Test func `Cancelling a test while evaluating traits skips the test`() async { + await testCancellation(testSkipped: 1) { configuration in + await Test(CancelledTrait()) { + Issue.record("Recorded an issue!") + }.run(configuration: configuration) + } + } + + @Test func `Cancelling the current task while evaluating traits skips the test`() async { + await testCancellation(testSkipped: 1) { configuration in + await Test(CancelledTrait(cancelsTask: true)) { + Issue.record("Recorded an issue!") + }.run(configuration: configuration) + } + } + +#if !SWT_NO_EXIT_TESTS + @Test func `Cancelling the current test from within an exit test`() async { + await testCancellation(testCancelled: 1, testCaseCancelled: 1) { configuration in + await Test { + await #expect(processExitsWith: .success) { + try Test.cancel("Cancelled test") + } + #expect(Task.isCancelled) + try Task.checkCancellation() + }.run(configuration: configuration) + } + } + + @Test func `Cancelling the current test case from within an exit test`() async { + await testCancellation(testCancelled: 1, testCaseCancelled: 1) { configuration in + await Test { + await #expect(processExitsWith: .success) { + try Test.Case.cancel("Cancelled test") + } + #expect(Task.isCancelled) + try Task.checkCancellation() + }.run(configuration: configuration) + } + } + + @Test func `Cancelling the current task in an exit test doesn't cancel the test`() async { + await testCancellation(testCancelled: 0, testCaseCancelled: 0) { configuration in + await Test { + await #expect(processExitsWith: .success) { + withUnsafeCurrentTask { $0?.cancel() } + } + #expect(!Task.isCancelled) + try Task.checkCancellation() + }.run(configuration: configuration) + } + } +#endif +} + +#if canImport(XCTest) +import XCTest + +final class TestCancellationTests: XCTestCase { + func testCancellationFromBackgroundTask() async { + let testCancelled = expectation(description: "Test cancelled") + testCancelled.isInverted = true + + let issueRecorded = expectation(description: "Issue recorded") + + var configuration = Configuration() + configuration.eventHandler = { event, _ in + if case .testCancelled = event.kind { + testCancelled.fulfill() + } else if case .issueRecorded = event.kind { + issueRecorded.fulfill() + } + } + + await Test { + await Task.detached { + _ = try? Test.cancel("Cancelled test") + }.value + }.run(configuration: configuration) + + await fulfillment(of: [testCancelled, issueRecorded], timeout: 0.0) + } +} +#endif + +// MARK: - Fixtures + +struct CancelledTrait: TestTrait { + var throwsWithoutCancelling = false + var cancelsTask = false + + func prepare(for test: Test) async throws { + if throwsWithoutCancelling { + throw CancellationError() + } + if cancelsTask { + withUnsafeCurrentTask { $0?.cancel() } + try Task.checkCancellation() + } + try Test.cancel("Cancelled from trait") + } +} + +#if !SWT_NO_SNAPSHOT_TYPES +struct `Shows as skipped in Xcode 16` { + @Test func `Cancelled test`() throws { + try Test.cancel("This test should appear cancelled/skipped") + } +} +#endif diff --git a/Tests/TestingTests/TestSupport/TestingAdditions.swift b/Tests/TestingTests/TestSupport/TestingAdditions.swift index 5c596785e..30d53ce7f 100644 --- a/Tests/TestingTests/TestSupport/TestingAdditions.swift +++ b/Tests/TestingTests/TestSupport/TestingAdditions.swift @@ -187,7 +187,9 @@ extension Test { init( _ traits: any TestTrait..., arguments collection: C, - parameters: [Parameter] = [], + parameters: [Parameter] = [ + Parameter(index: 0, firstName: "x", type: C.Element.self), + ], sourceLocation: SourceLocation = #_sourceLocation, column: Int = #column, name: String = #function, @@ -216,7 +218,10 @@ extension Test { init( _ traits: any TestTrait..., arguments collection1: C1, _ collection2: C2, - parameters: [Parameter] = [], + parameters: [Parameter] = [ + Parameter(index: 0, firstName: "x", type: C1.Element.self), + Parameter(index: 1, firstName: "y", type: C2.Element.self), + ], sourceLocation: SourceLocation = #_sourceLocation, name: String = #function, testFunction: @escaping @Sendable (C1.Element, C2.Element) async throws -> Void @@ -239,7 +244,10 @@ extension Test { init( _ traits: any TestTrait..., arguments zippedCollections: Zip2Sequence, - parameters: [Parameter] = [], + parameters: [Parameter] = [ + Parameter(index: 0, firstName: "x", type: C1.Element.self), + Parameter(index: 1, firstName: "y", type: C2.Element.self), + ], sourceLocation: SourceLocation = #_sourceLocation, name: String = #function, testFunction: @escaping @Sendable ((C1.Element, C2.Element)) async throws -> Void