Record issues generated within exit tests.

This PR introduces a "back channel" file handle to exit tests, allowing us to
record issues that occur within exit test bodies. For example:

```swift
await #expect(exitsWith: .failure) {
  let context = try #require(possiblyMissingContext)
  ...
}
```

In this example, if the call to `try #require()` finds `nil`, it will record an
issue, but that issue today will be lost because there's no mechanism to forward
the issue back to the parent process hosting the exit test. This PR fixes that!

Issues are converted to JSON using the same schema we use for event handling,
then written over a pipe back to the parent process where they are decoded. This
decoding is lossy, so there will be further refinement needed here to try to
preserve more information about the recorded issues. That said, "it's got good
bones" right?

On Darwin, Linux, and FreeBSD, the pipe's write end is allowed to survive into
the child process (i.e. no `FD_CLOEXEC`). On Windows, the equivalent is to tell
`CreateProcessW()` to explicitly inherit a `HANDLE`. The identity of this file
descriptor or handle is passed to the child process via environment variable.
The child process then parses the file descriptor or handle out of the
environment and converts it back to a `FileHandle` that is then connected to an
instance of `Configuration` with an event handler set, and off we go.

Because we can now report these issues back to the parent process, I've removed
the compile-time diagnostic in the `#expect(exitsWith:)` macro implementation
that we emit when we see a nested `#expect()` or `#require()` call.

Author: grynspan

Sources/Testing/ABI/EntryPoints/ABIEntryPoint.swift

@@ -120,7 +120,7 @@ private func entryPoint(
     args?.eventStreamVersion = eventStreamVersionIfNil
   }
 
-  let eventHandler = try eventHandlerForStreamingEvents(version: args?.eventStreamVersion, forwardingTo: recordHandler)
+  let eventHandler = try eventHandlerForStreamingEvents(version: args?.eventStreamVersion, encodeAsJSONLines: false, forwardingTo: recordHandler)
   let exitCode = await entryPoint(passing: args, eventHandler: eventHandler)
 
   // To maintain compatibility with Xcode 16 Beta 1, suppress custom exit codes.

Sources/Testing/ABI/EntryPoints/EntryPoint.swift

@@ -468,8 +468,11 @@ public func configurationForEntryPoint(from args: __CommandLineArguments_v0) thr
   // Event stream output (experimental)
   if let eventStreamOutputPath = args.eventStreamOutputPath {
     let file = try FileHandle(forWritingAtPath: eventStreamOutputPath)
-    let eventHandler = try eventHandlerForStreamingEvents(version: args.eventStreamVersion) { json in
-      try? _writeJSONLine(json, to: file)
+    let eventHandler = try eventHandlerForStreamingEvents(version: args.eventStreamVersion, encodeAsJSONLines: true) { json in
+      _ = try? file.withLock {
+        try file.write(json)
+        try file.write("\n")
+      }
     }
     configuration.eventHandler = { [oldEventHandler = configuration.eventHandler] event, context in
       eventHandler(event, context)
@@ -536,13 +539,20 @@ public func configurationForEntryPoint(from args: __CommandLineArguments_v0) thr
 ///
 /// - Parameters:
 ///   - version: The ABI version to use.
+///   - encodeAsJSONLines: Whether or not to ensure JSON passed to
+///     `eventHandler` is encoded as JSON Lines (i.e. that it does not contain
+///     extra newlines.)
 ///   - eventHandler: The event handler to forward encoded events to. The
 ///     encoding of events depends on `version`.
 ///
 /// - Returns: An event handler.
 ///
 /// - Throws: If `version` is not a supported ABI version.
-func eventHandlerForStreamingEvents(version: Int?, forwardingTo eventHandler: @escaping @Sendable (UnsafeRawBufferPointer) -> Void) throws -> Event.Handler {
+func eventHandlerForStreamingEvents(
+  version: Int?,
+  encodeAsJSONLines: Bool,
+  forwardingTo eventHandler: @escaping @Sendable (UnsafeRawBufferPointer) -> Void
+) throws -> Event.Handler {
   switch version {
 #if !SWT_NO_SNAPSHOT_TYPES
   case -1:
@@ -551,57 +561,11 @@ func eventHandlerForStreamingEvents(version: Int?, forwardingTo eventHandler: @e
     eventHandlerForStreamingEventSnapshots(to: eventHandler)
 #endif
   case nil, 0:
-    ABIv0.Record.eventHandler(forwardingTo: eventHandler)
+    ABIv0.Record.eventHandler(encodeAsJSONLines: encodeAsJSONLines, forwardingTo: eventHandler)
   case let .some(unsupportedVersion):
     throw _EntryPointError.invalidArgument("--event-stream-version", value: "\(unsupportedVersion)")
   }
 }
-
-/// Post-process encoded JSON and write it to a file.
-///
-/// - Parameters:
-///   - json: The JSON to write.
-///   - file: The file to write to.
-///
-/// - Throws: Whatever is thrown when writing to `file`.
-private func _writeJSONLine(_ json: UnsafeRawBufferPointer, to file: borrowing FileHandle) throws {
-  func isASCIINewline(_ byte: UInt8) -> Bool {
-    byte == UInt8(ascii: "\r") || byte == UInt8(ascii: "\n")
-  }
-
-  func write(_ json: UnsafeRawBufferPointer) throws {
-    try file.withLock {
-      try file.write(json)
-      try file.write("\n")
-    }
-  }
-
-  // We don't actually expect the JSON encoder to produce output containing
-  // newline characters, so in debug builds we'll log a diagnostic message.
-  if _slowPath(json.contains(where: isASCIINewline)) {
-#if DEBUG
-    let message = Event.ConsoleOutputRecorder.warning(
-      "JSON encoder produced one or more newline characters while encoding an event to JSON. Please file a bug report at https://github.com/swiftlang/swift-testing/issues/new",
-      options: .for(.stderr)
-    )
-#if SWT_TARGET_OS_APPLE
-    try? FileHandle.stderr.write(message)
-#else
-    print(message)
-#endif
-#endif
-
-    // Remove the newline characters to conform to JSON lines specification.
-    var json = Array(json)
-    json.removeAll(where: isASCIINewline)
-    try json.withUnsafeBytes { json in
-      try write(json)
-    }
-  } else {
-    // No newlines found, no need to copy the buffer.
-    try write(json)
-  }
-}
 #endif
 
 // MARK: - Command-line interface options

Sources/Testing/ABI/v0/ABIv0.Record+Streaming.swift

@@ -10,10 +10,46 @@
 
 #if canImport(Foundation) && (!SWT_NO_FILE_IO || !SWT_NO_ABI_ENTRY_POINT)
 extension ABIv0.Record {
+  /// Post-process encoded JSON and write it to a file.
+  ///
+  /// - Parameters:
+  ///   - json: The JSON to write.
+  ///   - file: The file to write to.
+  ///
+  /// - Throws: Whatever is thrown when writing to `file`.
+  private static func _asJSONLine(_ json: UnsafeRawBufferPointer, _ eventHandler: (_ recordJSON: UnsafeRawBufferPointer) throws -> Void) rethrows {
+    // We don't actually expect the JSON encoder to produce output containing
+    // newline characters, so in debug builds we'll log a diagnostic message.
+    if _slowPath(json.contains(where: \.isASCIINewline)) {
+  #if DEBUG
+      let message = Event.ConsoleOutputRecorder.warning(
+        "JSON encoder produced one or more newline characters while encoding an event to JSON. Please file a bug report at https://github.com/swiftlang/swift-testing/issues/new",
+        options: .for(.stderr)
+      )
+  #if SWT_TARGET_OS_APPLE
+      try? FileHandle.stderr.write(message)
+  #else
+      print(message)
+  #endif
+  #endif
+
+      // Remove the newline characters to conform to JSON lines specification.
+      var json = Array(json)
+      json.removeAll(where: \.isASCIINewline)
+      try json.withUnsafeBytes(eventHandler)
+    } else {
+      // No newlines found, no need to copy the buffer.
+      try eventHandler(json)
+    }
+  }
+
   /// Create an event handler that encodes events as JSON and forwards them to
   /// an ABI-friendly event handler.
   ///
   /// - Parameters:
+  ///   - encodeAsJSONLines: Whether or not to ensure JSON passed to
+  ///     `eventHandler` is encoded as JSON Lines (i.e. that it does not contain
+  ///     extra newlines.)
   ///   - eventHandler: The event handler to forward events to. See
   ///     ``ABIv0/EntryPoint-swift.typealias`` for more information.
   ///
@@ -27,10 +63,17 @@ extension ABIv0.Record {
   /// performs additional postprocessing before writing JSON data to ensure it
   /// does not contain any newline characters.
   static func eventHandler(
+    encodeAsJSONLines: Bool,
     forwardingTo eventHandler: @escaping @Sendable (_ recordJSON: UnsafeRawBufferPointer) -> Void
   ) -> Event.Handler {
+    // Encode as JSON Lines if requested.
+    var eventHandlerCopy = eventHandler
+    if encodeAsJSONLines {
+      eventHandlerCopy = { @Sendable in _asJSONLine($0, eventHandler) }
+    }
+
     let humanReadableOutputRecorder = Event.HumanReadableOutputRecorder()
-    return { event, context in
+    return { [eventHandler = eventHandlerCopy] event, context in
       if case .testDiscovered = event.kind, let test = context.test {
         try? JSON.withEncoding(of: Self(encoding: test)) { testJSON in
           eventHandler(testJSON)

Sources/Testing/ExitTests/ExitTest.swift

@@ -66,6 +66,32 @@ public struct ExitTest: Sendable {
 #endif
   }
 
+  /// Find a back channel file handle set up by the parent process.
+  ///
+  /// - Returns: A file handle open for writing to which events should be
+  ///   written, or `nil` if the file handle could not be resolved.
+  private static func _findBackChannel() -> FileHandle? {
+    guard let backChannelEnvironmentVariable = Environment.variable(named: "SWT_EXPERIMENTAL_BACKCHANNEL_FD") else {
+      return nil
+    }
+
+    var fd: CInt?
+#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD)
+    fd = CInt(backChannelEnvironmentVariable).map(dup)
+#elseif os(Windows)
+    if let handle = UInt(backChannelEnvironmentVariable).flatMap(HANDLE.init(bitPattern:)) {
+      fd = _open_osfhandle(Int(bitPattern: handle), _O_WRONLY | _O_BINARY)
+    }
+#else
+#warning("Platform-specific implementation missing: back-channel pipe unavailable")
+#endif
+    guard let fd, fd >= 0 else {
+      return nil
+    }
+
+    return try? FileHandle(unsafePOSIXFileDescriptor: fd, mode: "wb")
+  }
+
   /// Call the exit test in the current process.
   ///
   /// This function invokes the closure originally passed to
@@ -76,8 +102,27 @@ public struct ExitTest: Sendable {
   public func callAsFunction() async -> Never {
     Self._disableCrashReporting()
 
+    // Set up the configuration for this process.
+    var configuration = Configuration()
+    if let backChannel = Self._findBackChannel() {
+      // Encode events as JSON and write them to the back channel file handle.
+      var eventHandler = ABIv0.Record.eventHandler(encodeAsJSONLines: true) { json in
+        try? backChannel.write(json)
+      }
+
+      // Only forward issue-recorded events. (If we start handling other kinds
+      // of event in the future, we can forward them too.)
+      eventHandler = { [eventHandler] event, eventContext in
+        if case .issueRecorded = event.kind {
+          eventHandler(event, eventContext)
+        }
+      }
+
+      configuration.eventHandler = eventHandler
+    }
+
     do {
-      try await body()
+      try await Configuration.withCurrent(configuration, perform: body)
     } catch {
       _errorInMain(error)
     }
@@ -342,11 +387,109 @@ extension ExitTest {
         childEnvironment["SWT_EXPERIMENTAL_EXIT_TEST_SOURCE_LOCATION"] = String(decoding: json, as: UTF8.self)
       }
 
-      return try await spawnAndWait(
-        forExecutableAtPath: childProcessExecutablePath,
-        arguments: childArguments,
-        environment: childEnvironment
-      )
+      return try await withThrowingTaskGroup(of: ExitCondition?.self) { taskGroup in
+        // Create a "back channel" pipe to handle events from the child process.
+        let backChannel = try FileHandle.Pipe()
+
+        // Let the child process know how to find the back channel by setting a
+        // known environment variable to the corresponding file descriptor
+        // (HANDLE on Windows.)
+        var backChannelEnvironmentVariable: String?
+#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD)
+        backChannelEnvironmentVariable = backChannel.writeEnd.withUnsafePOSIXFileDescriptor { fd in
+          fd.map(String.init(describing:))
+        }
+#elseif os(Windows)
+        backChannelEnvironmentVariable = backChannel.writeEnd.withUnsafeWindowsHANDLE { handle in
+          handle.flatMap { String(describing: UInt(bitPattern: $0)) }
+        }
+#else
+#warning("Platform-specific implementation missing: back-channel pipe unavailable")
+#endif
+        if let backChannelEnvironmentVariable {
+          childEnvironment["SWT_EXPERIMENTAL_BACKCHANNEL_FD"] = backChannelEnvironmentVariable
+        }
+
+        // Spawn the child process.
+        let processID = try withUnsafePointer(to: backChannel.writeEnd) { writeEnd in
+          try spawnExecutable(
+            atPath: childProcessExecutablePath,
+            arguments: childArguments,
+            environment: childEnvironment,
+            additionalFileHandles: .init(start: writeEnd, count: 1)
+          )
+        }
+
+        // Await termination of the child process.
+        taskGroup.addTask {
+          try await wait(for: processID)
+        }
+
+        // Read back all data written to the back channel by the child process
+        // and process it as a (minimal) event stream.
+        let readEnd = backChannel.closeWriteEnd()
+        taskGroup.addTask {
+          Self._processRecordsFromBackChannel(readEnd)
+          return nil
+        }
+
+        // This is a roundabout way of saying "and return the exit condition
+        // yielded by wait(for:)".
+        return try await taskGroup.compactMap { $0 }.first { _ in true }!
+      }
+    }
+  }
+
+  /// Read lines from the given back channel file handle and process them as
+  /// event records.
+  ///
+  /// - Parameters:
+  ///   - backChannel: The file handle to read from. Reading continues until an
+  ///     error is encountered or the end of the file is reached.
+  private static func _processRecordsFromBackChannel(_ backChannel: borrowing FileHandle) {
+    let bytes: [UInt8]
+    do {
+      bytes = try backChannel.readToEnd()
+    } catch {
+      // NOTE: an error caught here indicates an I/O problem.
+      // TODO: should we record these issues as systemic instead?
+      Issue.record(error)
+      return
+    }
+
+    for recordJSON in bytes.split(whereSeparator: \.isASCIINewline) where !recordJSON.isEmpty {
+      do {
+        try recordJSON.withUnsafeBufferPointer { recordJSON in
+          try Self._processRecord(.init(recordJSON), fromBackChannel: backChannel)
+        }
+      } catch {
+        // NOTE: an error caught here indicates a decoding problem.
+        // TODO: should we record these issues as systemic instead?
+        Issue.record(error)
+      }
+    }
+  }
+
+  /// Decode a line of JSON read from a back channel file handle and handle it
+  /// as if the corresponding event occurred locally.
+  ///
+  /// - Parameters:
+  ///   - recordJSON: The JSON to decode and process.
+  ///   - backChannel: The file handle that `recordJSON` was read from.
+  ///
+  /// - Throws: Any error encountered attempting to decode or process the JSON.
+  private static func _processRecord(_ recordJSON: UnsafeRawBufferPointer, fromBackChannel backChannel: borrowing FileHandle) throws {
+    let record = try JSON.decode(ABIv0.Record.self, from: recordJSON)
+
+    if case let .event(event) = record.kind, let issue = event.issue {
+      // Translate the issue back into a "real" issue and record it
+      // in the parent process. This translation is, of course, lossy
+      // due to the process boundary, but we make a best effort.
+      let comments: [Comment] = event.messages.compactMap { message in
+        message.symbol == .details ? Comment(rawValue: message.text) : nil
+      }
+      let issue = Issue(kind: .unconditional, comments: comments, sourceContext: .init(backtrace: nil, sourceLocation: issue.sourceLocation))
+      issue.record()
     }
   }
 }