Skip to content

Commit 64c8ce6

Browse files
grynspangrynspan
authored
Add experimental SPI to cancel a running test. (#1284)
This PR introduces `Test.cancel()` and `Test.Case.cancel()` which cancel the current test/suite and the current test case, respectively. For example: ```swift @test(arguments: [Food.burger, .fries, .iceCream]) func `Food truck is well-stocked`(_ food: Food) throws { if food == .iceCream && Season.current == .winter { try Test.Case.cancel("It's too cold for ice cream.") } // ... } ``` These functions work by cancelling the child task associated with the current test or test case, then throwing an error to end local execution early. Compare `XCTSkip()` which, in Swift, is just a thrown error that the XCTest harness special-cases, or `XCTSkip()` in Objective-C which actually throws an exception to force the caller to exit early. Resolves #120. Resolves #1289. Resolves rdar://159150449. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated.
1 parent 78f6ced commit 64c8ce6

File tree

15 files changed

+839
-86
lines changed

15 files changed

+839
-86
lines changed

‎Sources/Testing/ABI/Encoded/ABI.EncodedAttachment.swift‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -38,12 +38,6 @@ extension ABI {
3838
/// - Warning: Inline attachment content is not yet part of the JSON schema.
3939
var _bytes: Bytes?
4040

41-
/// The source location where this attachment was created.
42-
///
43-
/// - Warning: Attachment source locations are not yet part of the JSON
44-
/// schema.
45-
var _sourceLocation: SourceLocation?
46-
4741
init(encoding attachment: borrowing Attachment<AnyAttachable>, in eventContext: borrowing Event.Context) {
4842
path = attachment.fileSystemPath
4943

@@ -55,8 +49,6 @@ extension ABI {
5549
return Bytes(rawValue: [UInt8](bytes))
5650
}
5751
}
58-
59-
_sourceLocation = attachment.sourceLocation
6052
}
6153
}
6254

‎Sources/Testing/ABI/Encoded/ABI.EncodedEvent.swift‎

Lines changed: 48 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,10 @@ extension ABI {
2929
case issueRecorded
3030
case valueAttached
3131
case testCaseEnded
32+
case testCaseCancelled = "_testCaseCancelled"
3233
case testEnded
3334
case testSkipped
35+
case testCancelled = "_testCancelled"
3436
case runEnded
3537
}
3638

@@ -64,6 +66,38 @@ extension ABI {
6466
/// - Warning: Test cases are not yet part of the JSON schema.
6567
var _testCase: EncodedTestCase<V>?
6668

69+
/// The comments the test author provided for this event, if any.
70+
///
71+
/// The value of this property contains the comments related to the primary
72+
/// user action that caused this event to be generated.
73+
///
74+
/// Some kinds of events have additional associated comments. For example,
75+
/// when using ``withKnownIssue(_:isIntermittent:sourceLocation:_:)``, there
76+
/// can be separate comments for the "underlying" issue versus the known
77+
/// issue matcher, and either can be `nil`. In such cases, the secondary
78+
/// comment(s) are represented via a distinct property depending on the kind
79+
/// of that event.
80+
///
81+
/// - Warning: Comments at this level are not yet part of the JSON schema.
82+
var _comments: [String]?
83+
84+
/// A source location associated with this event, if any.
85+
///
86+
/// The value of this property represents the source location most closely
87+
/// related to the primary user action that caused this event to be
88+
/// generated.
89+
///
90+
/// Some kinds of events have additional associated source locations. For
91+
/// example, when using ``withKnownIssue(_:isIntermittent:sourceLocation:_:)``,
92+
/// there can be separate source locations for the "underlying" issue versus
93+
/// the known issue matcher. In such cases, the secondary source location(s)
94+
/// are represented via a distinct property depending on the kind of that
95+
/// event.
96+
///
97+
/// - Warning: Source locations at this level of the JSON schema are not yet
98+
/// part of said JSON schema.
99+
var _sourceLocation: SourceLocation?
100+
67101
init?(encoding event: borrowing Event, in eventContext: borrowing Event.Context, messages: borrowing [Event.HumanReadableOutputRecorder.Message]) {
68102
switch event.kind {
69103
case .runStarted:
@@ -78,18 +112,31 @@ extension ABI {
78112
case let .issueRecorded(recordedIssue):
79113
kind = .issueRecorded
80114
issue = EncodedIssue(encoding: recordedIssue, in: eventContext)
115+
_comments = recordedIssue.comments.map(\.rawValue)
116+
_sourceLocation = recordedIssue.sourceLocation
81117
case let .valueAttached(attachment):
82118
kind = .valueAttached
83119
self.attachment = EncodedAttachment(encoding: attachment, in: eventContext)
120+
_sourceLocation = attachment.sourceLocation
84121
case .testCaseEnded:
85122
if eventContext.test?.isParameterized == false {
86123
return nil
87124
}
88125
kind = .testCaseEnded
126+
case let .testCaseCancelled(skipInfo):
127+
kind = .testCaseCancelled
128+
_comments = Array(skipInfo.comment).map(\.rawValue)
129+
_sourceLocation = skipInfo.sourceLocation
89130
case .testEnded:
90131
kind = .testEnded
91-
case .testSkipped:
132+
case let .testSkipped(skipInfo):
92133
kind = .testSkipped
134+
_comments = Array(skipInfo.comment).map(\.rawValue)
135+
_sourceLocation = skipInfo.sourceLocation
136+
case let .testCancelled(skipInfo):
137+
kind = .testCancelled
138+
_comments = Array(skipInfo.comment).map(\.rawValue)
139+
_sourceLocation = skipInfo.sourceLocation
93140
case .runEnded:
94141
kind = .runEnded
95142
default:

‎Sources/Testing/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,7 @@ add_library(Testing
9393
Test.ID.Selection.swift
9494
Test.ID.swift
9595
Test.swift
96+
Test+Cancellation.swift
9697
Test+Discovery.swift
9798
Test+Discovery+Legacy.swift
9899
Test+Macro.swift

‎Sources/Testing/Events/Event.swift‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,18 @@ public struct Event: Sendable {
7474
/// that was passed to the event handler along with this event.
7575
case testCaseEnded
7676

77+
/// A test case was cancelled.
78+
///
79+
/// - Parameters:
80+
/// - skipInfo: A ``SkipInfo`` with details about the cancelled test case.
81+
///
82+
/// This event is generated by a call to ``Test/Case/cancel(_:sourceLocation:)``.
83+
///
84+
/// The test case that was cancelled is contained in the ``Event/Context``
85+
/// instance that was passed to the event handler along with this event.
86+
@_spi(Experimental)
87+
indirect case testCaseCancelled(_ skipInfo: SkipInfo)
88+
7789
/// An expectation was checked with `#expect()` or `#require()`.
7890
///
7991
/// - Parameters:
@@ -121,6 +133,18 @@ public struct Event: Sendable {
121133
/// available from this event's ``Event/testID`` property.
122134
indirect case testSkipped(_ skipInfo: SkipInfo)
123135

136+
/// A test was cancelled.
137+
///
138+
/// - Parameters:
139+
/// - skipInfo: A ``SkipInfo`` with details about the cancelled test.
140+
///
141+
/// This event is generated by a call to ``Test/cancel(_:sourceLocation:)``.
142+
///
143+
/// The test that was cancelled is contained in the ``Event/Context``
144+
/// instance that was passed to the event handler along with this event.
145+
@_spi(Experimental)
146+
indirect case testCancelled(_ skipInfo: SkipInfo)
147+
124148
/// A step in the runner plan ended.
125149
///
126150
/// - Parameters:
@@ -395,6 +419,18 @@ extension Event.Kind {
395419
/// A test case ended.
396420
case testCaseEnded
397421

422+
/// A test case was cancelled.
423+
///
424+
/// - Parameters:
425+
/// - skipInfo: A ``SkipInfo`` with details about the cancelled test case.
426+
///
427+
/// This event is generated by a call to ``Test/Case/cancel(_:sourceLocation:)``.
428+
///
429+
/// The test case that was cancelled is contained in the ``Event/Context``
430+
/// instance that was passed to the event handler along with this event.
431+
@_spi(Experimental)
432+
indirect case testCaseCancelled(_ skipInfo: SkipInfo)
433+
398434
/// An expectation was checked with `#expect()` or `#require()`.
399435
///
400436
/// - Parameters:
@@ -431,6 +467,18 @@ extension Event.Kind {
431467
/// - skipInfo: A ``SkipInfo`` containing details about this skipped test.
432468
indirect case testSkipped(_ skipInfo: SkipInfo)
433469

470+
/// A test was cancelled.
471+
///
472+
/// - Parameters:
473+
/// - skipInfo: A ``SkipInfo`` with details about the cancelled test.
474+
///
475+
/// This event is generated by a call to ``Test/cancel(_:sourceLocation:)``.
476+
///
477+
/// The test that was cancelled is contained in the ``Event/Context``
478+
/// instance that was passed to the event handler along with this event.
479+
@_spi(Experimental)
480+
indirect case testCancelled(_ skipInfo: SkipInfo)
481+
434482
/// A step in the runner plan ended.
435483
///
436484
/// - Parameters:
@@ -479,6 +527,8 @@ extension Event.Kind {
479527
self = .testCaseStarted
480528
case .testCaseEnded:
481529
self = .testCaseEnded
530+
case let .testCaseCancelled(skipInfo):
531+
self = .testCaseCancelled(skipInfo)
482532
case let .expectationChecked(expectation):
483533
let expectationSnapshot = Expectation.Snapshot(snapshotting: expectation)
484534
self = Snapshot.expectationChecked(expectationSnapshot)
@@ -490,6 +540,8 @@ extension Event.Kind {
490540
self = .testEnded
491541
case let .testSkipped(skipInfo):
492542
self = .testSkipped(skipInfo)
543+
case let .testCancelled(skipInfo):
544+
self = .testCancelled(skipInfo)
493545
case .planStepEnded:
494546
self = .planStepEnded
495547
case let .iterationEnded(index):

‎Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift‎

Lines changed: 50 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,9 @@ extension Event {
7373

7474
/// The number of known issues recorded for the test.
7575
var knownIssueCount = 0
76+
77+
/// Information about the cancellation of this test or test case.
78+
var cancellationInfo: SkipInfo?
7679
}
7780

7881
/// Data tracked on a per-test basis.
@@ -251,6 +254,7 @@ extension Event.HumanReadableOutputRecorder {
251254
0
252255
}
253256
let test = eventContext.test
257+
let testCase = eventContext.testCase
254258
let keyPath = eventContext.keyPath
255259
let testName = if let test {
256260
if let displayName = test.displayName {
@@ -310,6 +314,9 @@ extension Event.HumanReadableOutputRecorder {
310314
case .testCaseStarted:
311315
context.testData[keyPath] = .init(startInstant: instant)
312316

317+
case let .testCancelled(skipInfo), let .testCaseCancelled(skipInfo):
318+
context.testData[keyPath]?.cancellationInfo = skipInfo
319+
313320
default:
314321
// These events do not manipulate the context structure.
315322
break
@@ -404,21 +411,29 @@ extension Event.HumanReadableOutputRecorder {
404411
} else {
405412
""
406413
}
407-
return if issues.errorIssueCount > 0 {
408-
CollectionOfOne(
409-
Message(
410-
symbol: .fail,
411-
stringValue: "\(_capitalizedTitle(for: test)) \(testName)\(testCasesCount) failed after \(duration)\(issues.description)."
412-
)
413-
) + _formattedComments(for: test)
414+
var cancellationComment = "."
415+
let (symbol, verbed): (Event.Symbol, String)
416+
if issues.errorIssueCount > 0 {
417+
(symbol, verbed) = (.fail, "failed")
418+
} else if !test.isParameterized, let cancellationInfo = testData.cancellationInfo {
419+
if let comment = cancellationInfo.comment {
420+
cancellationComment = ": \"\(comment.rawValue)\""
421+
}
422+
(symbol, verbed) = (.skip, "was cancelled")
414423
} else {
415-
[
416-
Message(
417-
symbol: .pass(knownIssueCount: issues.knownIssueCount),
418-
stringValue: "\(_capitalizedTitle(for: test)) \(testName)\(testCasesCount) passed after \(duration)\(issues.description)."
419-
)
420-
]
424+
(symbol, verbed) = (.pass(knownIssueCount: issues.knownIssueCount), "passed")
425+
}
426+
427+
var result = [
428+
Message(
429+
symbol: symbol,
430+
stringValue: "\(_capitalizedTitle(for: test)) \(testName)\(testCasesCount) \(verbed) after \(duration)\(issues.description)\(cancellationComment)"
431+
)
432+
]
433+
if issues.errorIssueCount > 0 {
434+
result += _formattedComments(for: test)
421435
}
436+
return result
422437

423438
case let .testSkipped(skipInfo):
424439
let test = test!
@@ -443,7 +458,7 @@ extension Event.HumanReadableOutputRecorder {
443458
} else {
444459
0
445460
}
446-
let labeledArguments = if let testCase = eventContext.testCase {
461+
let labeledArguments = if let testCase {
447462
testCase.labeledArguments()
448463
} else {
449464
""
@@ -523,7 +538,7 @@ extension Event.HumanReadableOutputRecorder {
523538
return result
524539

525540
case .testCaseStarted:
526-
guard let testCase = eventContext.testCase, testCase.isParameterized, let arguments = testCase.arguments else {
541+
guard let testCase, testCase.isParameterized, let arguments = testCase.arguments else {
527542
break
528543
}
529544

@@ -535,7 +550,7 @@ extension Event.HumanReadableOutputRecorder {
535550
]
536551

537552
case .testCaseEnded:
538-
guard verbosity > 0, let testCase = eventContext.testCase, testCase.isParameterized, let arguments = testCase.arguments else {
553+
guard verbosity > 0, let test, let testCase, testCase.isParameterized, let arguments = testCase.arguments else {
539554
break
540555
}
541556

@@ -544,18 +559,28 @@ extension Event.HumanReadableOutputRecorder {
544559
let issues = _issueCounts(in: testDataGraph)
545560
let duration = testData.startInstant.descriptionOfDuration(to: instant)
546561

547-
let message = if issues.errorIssueCount > 0 {
548-
Message(
549-
symbol: .fail,
550-
stringValue: "Test case passing \(arguments.count.counting("argument")) \(testCase.labeledArguments(includingQualifiedTypeNames: verbosity > 0)) to \(testName) failed after \(duration)\(issues.description)."
551-
)
562+
var cancellationComment = "."
563+
let (symbol, verbed): (Event.Symbol, String)
564+
if issues.errorIssueCount > 0 {
565+
(symbol, verbed) = (.fail, "failed")
566+
} else if !test.isParameterized, let cancellationInfo = testData.cancellationInfo {
567+
if let comment = cancellationInfo.comment {
568+
cancellationComment = ": \"\(comment.rawValue)\""
569+
}
570+
(symbol, verbed) = (.skip, "was cancelled")
552571
} else {
572+
(symbol, verbed) = (.pass(knownIssueCount: issues.knownIssueCount), "passed")
573+
}
574+
return [
553575
Message(
554-
symbol: .pass(knownIssueCount: issues.knownIssueCount),
555-
stringValue: "Test case passing \(arguments.count.counting("argument")) \(testCase.labeledArguments(includingQualifiedTypeNames: verbosity > 0)) to \(testName) passed after \(duration)\(issues.description)."
576+
symbol: symbol,
577+
stringValue: "Test case passing \(arguments.count.counting("argument")) \(testCase.labeledArguments(includingQualifiedTypeNames: verbosity > 0)) to \(testName) \(verbed) after \(duration)\(issues.description)\(cancellationComment)"
556578
)
557-
}
558-
return [message]
579+
]
580+
581+
case .testCancelled, .testCaseCancelled:
582+
// Handled in .testEnded and .testCaseEnded
583+
break
559584

560585
case let .iterationEnded(index):
561586
guard let iterationStartInstant = context.iterationStartInstant else {

0 commit comments

Comments
 (0)