From 497f079130da4b43e0d2a023dbecda80f113f4ab Mon Sep 17 00:00:00 2001 From: Tony Allevato Date: Fri, 25 Mar 2022 12:07:37 -0700 Subject: [PATCH] Add an XCTest observer to `swift_test` targets that generates a JUnit-style XML log at the path in the `XML_OUTPUT_PATH` environment variable defined by Bazel. Due to differences between the open-source XCTest and Xcode's XCTest, only Darwin-based platforms can distinguish "skipped" tests from "passing" tests at this time (and even on that platform, it can only do so by referencing private APIs). PiperOrigin-RevId: 437304646 --- swift/internal/swift_test.bzl | 24 +- tools/test_discoverer/TestPrinter.swift | 4 + tools/test_observer/BUILD | 12 + .../test_observer/BazelXMLTestObserver.swift | 210 ++++++++++++++++++ .../BazelXMLTestObserverRegistration.swift | 31 +++ .../StringInterpolation+XMLEscaping.swift | 38 ++++ 6 files changed, 315 insertions(+), 4 deletions(-) create mode 100644 tools/test_observer/BUILD create mode 100644 tools/test_observer/BazelXMLTestObserver.swift create mode 100644 tools/test_observer/BazelXMLTestObserverRegistration.swift create mode 100644 tools/test_observer/StringInterpolation+XMLEscaping.swift diff --git a/swift/internal/swift_test.bzl b/swift/internal/swift_test.bzl index 01e18888e..a46dc435b 100644 --- a/swift/internal/swift_test.bzl +++ b/swift/internal/swift_test.bzl @@ -60,11 +60,16 @@ def _create_xctest_bundle(name, actions, binary): args.add(xctest_bundle.path) args.add(binary) + # When XCTest loads this bundle, it will create an instance of this class + # which will register the observer that writes the XML output. + plist = '{ NSPrincipalClass = "BazelXMLTestObserverRegistration"; }' + actions.run_shell( arguments = [args], command = ( 'mkdir -p "$1/Contents/MacOS" && ' + - 'cp "$2" "$1/Contents/MacOS"' + 'cp "$2" "$1/Contents/MacOS" && ' + + 'echo \'{}\' > "$1/Contents/Info.plist"'.format(plist) ), inputs = [binary], mnemonic = "SwiftCreateTestBundle", @@ -200,6 +205,7 @@ def _swift_test_impl(ctx): srcs = ctx.files.srcs extra_copts = [] + extra_deps = [] # If no sources were provided and we're not using `.xctest` bundling, assume # that we need to discover tests using symbol graphs. @@ -217,6 +223,9 @@ def _swift_test_impl(ctx): # The generated test runner uses `@main`. extra_copts = ["-parse-as-library"] + extra_deps = [ctx.attr._test_observer] + elif is_bundled: + extra_deps = [ctx.attr._test_observer] if srcs: module_name = ctx.attr.module_name @@ -226,7 +235,9 @@ def _swift_test_impl(ctx): _, compilation_outputs = swift_common.compile( actions = ctx.actions, additional_inputs = ctx.files.swiftc_inputs, - compilation_contexts = get_compilation_contexts(ctx.attr.deps), + compilation_contexts = get_compilation_contexts( + ctx.attr.deps + extra_deps, + ), copts = expand_locations( ctx, ctx.attr.copts, @@ -236,7 +247,7 @@ def _swift_test_impl(ctx): feature_configuration = feature_configuration, module_name = module_name, srcs = srcs, - swift_infos = get_providers(ctx.attr.deps, SwiftInfo), + swift_infos = get_providers(ctx.attr.deps + extra_deps, SwiftInfo), swift_toolchain = swift_toolchain, target_name = ctx.label.name, ) @@ -253,7 +264,7 @@ def _swift_test_impl(ctx): additional_linking_contexts = [malloc_linking_context(ctx)], cc_feature_configuration = cc_feature_configuration, compilation_outputs = compilation_outputs, - deps = ctx.attr.deps, + deps = ctx.attr.deps + extra_deps, grep_includes = ctx.file._grep_includes, name = ctx.label.name, output_type = "executable", @@ -336,6 +347,11 @@ swift_test = rule( ), executable = True, ), + "_test_observer": attr.label( + default = Label( + "@build_bazel_rules_swift//tools/test_observer", + ), + ), "_xctest_runner_template": attr.label( allow_single_file = True, default = Label( diff --git a/tools/test_discoverer/TestPrinter.swift b/tools/test_discoverer/TestPrinter.swift index e9bbb5148..03186c075 100644 --- a/tools/test_discoverer/TestPrinter.swift +++ b/tools/test_discoverer/TestPrinter.swift @@ -134,6 +134,7 @@ struct TestPrinter { /// Prints the main test runner to a Swift source file. func printTestRunner(toFileAt url: URL) { var contents = """ + import BazelTestObservation import XCTest @main @@ -153,6 +154,9 @@ struct TestPrinter { } contents += """ + if let xmlObserver = BazelXMLTestObserver.default { + XCTestObservationCenter.shared.addTestObserver(xmlObserver) + } XCTMain(tests) } } diff --git a/tools/test_observer/BUILD b/tools/test_observer/BUILD new file mode 100644 index 000000000..e7b519942 --- /dev/null +++ b/tools/test_observer/BUILD @@ -0,0 +1,12 @@ +load("//swift:swift.bzl", "swift_library") + +swift_library( + name = "test_observer", + srcs = [ + "BazelXMLTestObserver.swift", + "BazelXMLTestObserverRegistration.swift", + "StringInterpolation+XMLEscaping.swift", + ], + module_name = "BazelTestObservation", + visibility = ["//visibility:public"], +) diff --git a/tools/test_observer/BazelXMLTestObserver.swift b/tools/test_observer/BazelXMLTestObserver.swift new file mode 100644 index 000000000..7995382e1 --- /dev/null +++ b/tools/test_observer/BazelXMLTestObserver.swift @@ -0,0 +1,210 @@ +// Copyright 2022 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import Foundation +import XCTest + +/// An XCTest observer that generates an XML file in the format described by the +/// [JUnit test result schema](https://windyroad.com.au/dl/Open%20Source/JUnit.xsd). +public final class BazelXMLTestObserver: NSObject { + /// The file handle to which the XML content will be written. + private let fileHandle: FileHandle + + /// The current indentation to print before each line, as UTF-8 code units. + private var indentation: Data + + /// The default XML-generating XCTest observer, which determines the output file based on the + /// value of the `XML_OUTPUT_FILE` environment variable. + /// + /// If the `XML_OUTPUT_FILE` environment variable is not set or the file at that path could not be + /// created and opened for writing, the value of this property will be nil. + public static let `default`: BazelXMLTestObserver? = { + guard + let outputPath = ProcessInfo.processInfo.environment["XML_OUTPUT_FILE"], + FileManager.default.createFile(atPath: outputPath, contents: nil, attributes: nil), + let fileHandle = FileHandle(forWritingAtPath: outputPath) + else { + return nil + } + return .init(fileHandle: fileHandle) + }() + + /// Creates a new XML-generating XCTest observer that writes its content to the given file handle. + private init(fileHandle: FileHandle) { + self.fileHandle = fileHandle + self.indentation = Data() + } + + /// Writes the given string to the observer's file handle. + private func writeLine(_ string: S) { + if !indentation.isEmpty { + fileHandle.write(indentation) + } + fileHandle.write(string.data(using: .utf8)!) // Conversion to UTF-8 cannot fail. + fileHandle.write(Data([UInt8(ascii: "\n")])) + } + + /// Increases the current indentation level by two spaces. + private func indent() { + indentation.append(contentsOf: [UInt8(ascii: " "), UInt8(ascii: " ")]) + } + + /// Reduces the current indentation level by two spaces. + private func dedent() { + indentation.removeLast(2) + } + + /// Canonicalizes the name of the test case for printing into the XML file. + /// + /// The canonical name of the test is `TestClass.testMethod` (i.e., Swift-style syntax). When + /// running tests under the Objective-C runtime, the test cases will have Objective-C-style names + /// (i.e., `-[TestClass testMethod]`), so this method converts those to the desired syntax. + /// + /// Any test name that does not match one of those two syntaxes is returned unchanged. + private func canonicalizedName(of testCase: XCTestCase) -> String { + let name = testCase.name + guard name.hasPrefix("-[") && name.hasSuffix("]") else { + return name + } + + let trimmedName = name.dropFirst(2).dropLast() + guard let spaceIndex = trimmedName.lastIndex(of: " ") else { + return String(trimmedName) + } + + return "\(trimmedName[.."#) + writeLine("") + indent() + } + + public func testBundleDidFinish(_ testBundle: Bundle) { + dedent() + writeLine("") + } + + public func testSuiteWillStart(_ testSuite: XCTestSuite) { + writeLine( + #""#) + indent() + } + + public func testSuiteDidFinish(_ testSuite: XCTestSuite) { + dedent() + writeLine("") + } + + public func testCaseWillStart(_ testCase: XCTestCase) { + writeLine( + #""#) + indent() + } + + public func testCaseDidFinish(_ testCase: XCTestCase) { + dedent() + writeLine("") + } + + // On platforms with the Objective-C runtime, we use the richer `XCTIssue`-based APIs. Anywhere + // else, we're building with the open-source version of XCTest which has only the older + // `didFailWithDescription` API. + #if canImport(ObjectiveC) + public func testCase(_ testCase: XCTestCase, didRecord issue: XCTIssue) { + let tag: String + switch issue.type { + case .assertionFailure, .performanceRegression, .unmatchedExpectedFailure: + tag = "failure" + case .system, .thrownError, .uncaughtException: + tag = "error" + @unknown default: + tag = "failure" + } + + writeLine(#"<\#(tag) message="\#(xmlEscaping: issue.compactDescription)"/>"#) + } + #else + public func testCase( + _ testCase: XCTestCase, + didFailWithDescription description: String, + inFile filePath: String?, + atLine lineNumber: Int + ) { + let tag = description.hasPrefix(#"threw error ""#) ? "error" : "failure" + writeLine(#"<\#(tag) message="\#(xmlEscaping: description)"/>"#) + } + #endif +} + +// Hacks ahead! XCTest does not declare the methods that it uses to notify observers of skipped +// tests as part of the public `XCTestObservation` protocol. Instead, they are only available on +// various framework-internal protocols that XCTest checks for conformance against at runtime. +// +// On Darwin platforms, thanks to the Objective-C runtime, we can declare protocols with the same +// names in our library and implement those methods, and XCTest will call them so that we can log +// the skipped tests in our output. Note that we have to re-specify the protocol name in the `@objc` +// attribute to remove the module name for the runtime. +// +// On non-Darwin platforms, we don't have an escape hatch because XCTest is implemented in pure +// Swift and we can't play the same runtime games, so skipped tests simply get tracked as "passing" +// there. +#if canImport(ObjectiveC) + /// Declares the observation method that is called by XCTest in Xcode 12.5 when a test case is + /// skipped. + @objc(_XCTestObservationInternal) + protocol _XCTestObservationInternal { + func testCase( + _ testCase: XCTestCase, + wasSkippedWithDescription description: String, + sourceCodeContext: XCTSourceCodeContext?) + } + + extension BazelXMLTestObserver: _XCTestObservationInternal { + public func testCase( + _ testCase: XCTestCase, + wasSkippedWithDescription description: String, + sourceCodeContext: XCTSourceCodeContext? + ) { + self.testCase( + testCase, + didRecordSkipWithDescription: description, + sourceCodeContext: sourceCodeContext) + } + } + + /// Declares the observation method that is called by XCTest in Xcode 13 and later when a test + /// case is skipped. + @objc(_XCTestObservationPrivate) + protocol _XCTestObservationPrivate { + func testCase( + _ testCase: XCTestCase, + didRecordSkipWithDescription description: String, + sourceCodeContext: XCTSourceCodeContext?) + } + + extension BazelXMLTestObserver: _XCTestObservationPrivate { + public func testCase( + _ testCase: XCTestCase, + didRecordSkipWithDescription description: String, + sourceCodeContext: XCTSourceCodeContext? + ) { + writeLine(#""#) + } + } +#endif diff --git a/tools/test_observer/BazelXMLTestObserverRegistration.swift b/tools/test_observer/BazelXMLTestObserverRegistration.swift new file mode 100644 index 000000000..851b8806d --- /dev/null +++ b/tools/test_observer/BazelXMLTestObserverRegistration.swift @@ -0,0 +1,31 @@ +// Copyright 2022 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#if canImport(ObjectiveC) + import Foundation + import XCTest + + /// The principal class in an XCTest bundle on Darwin-based platforms, which registers the + /// XML-generating observer with the XCTest observation center when the bundle is loaded. + @objc(BazelXMLTestObserverRegistration) + public final class BazelXMLTestObserverRegistration: NSObject { + @objc public override init() { + super.init() + + if let observer = BazelXMLTestObserver.default { + XCTestObservationCenter.shared.addTestObserver(observer) + } + } + } +#endif diff --git a/tools/test_observer/StringInterpolation+XMLEscaping.swift b/tools/test_observer/StringInterpolation+XMLEscaping.swift new file mode 100644 index 000000000..47fe1d198 --- /dev/null +++ b/tools/test_observer/StringInterpolation+XMLEscaping.swift @@ -0,0 +1,38 @@ +// Copyright 2022 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +extension String.StringInterpolation { + /// Appends the given string to a string interpolation, escaping any characters with special XML + /// meanings. + mutating func appendInterpolation(xmlEscaping string: S) { + var remainder = string[...] + while let escapeIndex = remainder.firstIndex(where: { xmlEscapeMapping[$0] != nil }) { + appendLiteral(String(remainder[..": ">", +]