Skip to content
Merged
Show file tree
Hide file tree
Changes from 7 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,9 @@ extension Event {

/// The number of known issues recorded for the test.
var knownIssueCount = 0

/// The number of test cases for the test.
var testCasesCount: Int = 0
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: can omit the explicit type since this isn't a public property, to match prevailing style

Suggested change
var testCasesCount: Int = 0
var testCasesCount = 0

}

/// Data tracked on a per-test basis.
Expand Down Expand Up @@ -146,6 +149,29 @@ extension Event.HumanReadableOutputRecorder {

return (errorIssueCount, warningIssueCount, knownIssueCount, totalIssueCount, description)
}

/// Returns a formatted string describing the number of arguments in a test,
/// based on verbosity level.
///
/// - Parameters:
/// - isParameterized: To check test cases count should be included.
/// - count: The number of test cases in the test.
/// - verbosity: The verbosity level. If it's very verbose or higher,
/// the test cases count might get included.
///
///
/// - Returns: A string describing the number of test cases in the test,
/// or an empty string if it's not very verbose level.
///
private func _includeNumberOfTestCasesIfNeeded(
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You probably don't need this helper function and can just construct the string in place in the .testEnded handler below.

_ isParameterized: Bool,
count testCasesCount: Int,
verbosity verbose: Int
) -> String {
guard verbose >= 2, isParameterized else { return "" }
return " with \(testCasesCount.counting("test case"))"
}

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nitpick: stray newline? :)

}

/// Generate a title for the specified test (either "Test" or "Suite"),
Expand Down Expand Up @@ -281,6 +307,18 @@ extension Event.HumanReadableOutputRecorder {
testData.issueCount[issue.severity] = issueCount + 1
}
context.testData[id] = testData

case .testCaseStarted:
guard verbosity >= 2 else { break }
let id: [String] = if let test {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We don't check verbosity in this first switch intentionally so that the data we collect is always comprehensive. The second switch below checks verbosity before producing any messages.

test will never be nil for this event, and .testStarted will always be recorded before .testCaseStarted, so you can just write:

case .testCaseStarted:
  let test = test!
  context.testData[test.id.keyPathRepresentation]?.testCasesCount += 1

test.id.keyPathRepresentation
} else {
[]
}
var testData = context.testData[id] ?? .init(startInstant: instant)
testData.testCasesCount += 1
context.testData[id] = testData


default:
// These events do not manipulate the context structure.
Expand Down Expand Up @@ -366,18 +404,21 @@ extension Event.HumanReadableOutputRecorder {
let testData = testDataGraph?.value ?? .init(startInstant: instant)
let issues = _issueCounts(in: testDataGraph)
let duration = testData.startInstant.descriptionOfDuration(to: instant)
let testCasesCountMessage = _includeNumberOfTestCasesIfNeeded(test.isParameterized,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Here, you can probably just write:

let testCasesCount = if verbosity >= 2 && test.isParameterized {
  " with \(testData.testCasesCount.counting("test case"))"
} else {
  ""
}

("Message" has a domain-specific meaning in this file, which is why I renamed this variable in my example above.)

count: testData.testCasesCount,
verbosity: verbosity)
return if issues.errorIssueCount > 0 {
CollectionOfOne(
Message(
symbol: .fail,
stringValue: "\(_capitalizedTitle(for: test)) \(testName) failed after \(duration)\(issues.description)."
stringValue: "\(_capitalizedTitle(for: test)) \(testName)\(testCasesCountMessage) failed after \(duration)\(issues.description)."
)
) + _formattedComments(for: test)
} else {
[
Message(
symbol: .pass(knownIssueCount: issues.knownIssueCount),
stringValue: "\(_capitalizedTitle(for: test)) \(testName) passed after \(duration)\(issues.description)."
stringValue: "\(_capitalizedTitle(for: test)) \(testName)\(testCasesCountMessage) passed after \(duration)\(issues.description)."
)
]
}
Expand Down
45 changes: 45 additions & 0 deletions Tests/TestingTests/EventRecorderTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,45 @@ struct EventRecorderTests {
.first != nil
)
}

@available(_regexAPI, *)
@Test(
"number of arguments based on verbosity level at the end of test run",
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The display name of this test can be rephrased now to not mention verbosity. I'd also prefer if we could capitalize the first word to match prevailing style.

arguments: [
("f()", #".* Test f\(\) failed after .*"# , 0),
("f()", #".* Test f\(\) failed after .*"# , 2),
("d(_:)", #".* Test d\(_:\) with .+ test cases passed after.*"# , 2),
("PredictablyFailingTests", #".* Suite PredictablyFailingTests failed after .*"# , 1),
("PredictablyFailingTests", #".* Suite PredictablyFailingTests failed after .*"# , 2),
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: whitespace

Suggested change
("f()", #".* Test f\(\) failed after .*"# , 0),
("f()", #".* Test f\(\) failed after .*"# , 2),
("d(_:)", #".* Test d\(_:\) with .+ test cases passed after.*"# , 2),
("PredictablyFailingTests", #".* Suite PredictablyFailingTests failed after .*"# , 1),
("PredictablyFailingTests", #".* Suite PredictablyFailingTests failed after .*"# , 2),
("f()", #".* Test f\(\) failed after .*"#, 0),
("f()", #".* Test f\(\) failed after .*"#, 2),
("d(_:)", #".* Test d\(_:\) with .+ test cases passed after.*"#, 2),
("PredictablyFailingTests", #".* Suite PredictablyFailingTests failed after .*"#, 1),
("PredictablyFailingTests", #".* Suite PredictablyFailingTests failed after .*"#, 2),

]
)
func numberOfArgumentsAtTheEndOfTests(testName: String, expectedPattern: String, verbosity: Int) async throws {
let stream = Stream()

var configuration = Configuration()
let eventRecorder = Event.ConsoleOutputRecorder(writingUsing: stream.write)
configuration.eventHandler = { event, context in
eventRecorder.record(event, in: context)
}
configuration.verbosity = verbosity

await runTest(for: PredictablyFailingTests.self, configuration: configuration)

let buffer = stream.buffer.rawValue
if testsWithSignificantIOAreEnabled {
print(buffer, terminator: "")
}

let aurgmentRegex = try Regex(expectedPattern)

#expect(
(try buffer
.split(whereSeparator: \.isNewline)
.compactMap(aurgmentRegex.wholeMatch(in:))
.first) != nil
)
}


@available(_regexAPI, *)
@Test(
Expand Down Expand Up @@ -536,6 +575,11 @@ struct EventRecorderTests {
#expect(Bool(false))
}
}

@Test(.hidden, arguments: [1, 2, 3])
func d(_ arg: Int) {
#expect(arg > 0)
}

@Test(.hidden) func g() {
#expect(Bool(false))
Expand Down Expand Up @@ -566,3 +610,4 @@ struct EventRecorderTests {
}
}
}

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nitpick: stray newline?