WIP: Don’t stop on error

Author: ahoppen

SourceKitStressTester/Sources/StressTester/StressTester.swift

@@ -41,7 +41,7 @@ public class StressTester {
     }
   }
 
-  public func run(swiftc: String, compilerArgs: CompilerArgs) throws {
+  public func run(swiftc: String, compilerArgs: CompilerArgs) -> [Error] {
     var document = SourceKitDocument(swiftc: swiftc,
                                      args: compilerArgs,
                                      tempDir: options.tempDir,
@@ -59,20 +59,36 @@ public class StressTester {
     }
 
     // compute the actions for the entire tree
-    let (tree, _) = try document.open(rewriteMode: options.rewriteMode)
+    let tree: SourceFileSyntax
+    do {
+      tree = try document.open(rewriteMode: options.rewriteMode).0
+    } catch {
+      return [error]
+    }
     let (actions, priorActions) = computeActions(from: tree)
 
+    var errors: [Error] = []
+
     if let dryRunAction = options.dryRun {
-      try dryRunAction(actions)
-      return
+      do {
+        try dryRunAction(actions)
+      } catch {
+        errors.append(error)
+      }
+      return errors
     }
 
     if !priorActions.isEmpty {
-      // Update initial state
-      _ = try document.update() { sourceState in
-        for case .replaceText(let offset, let length, let text) in priorActions {
-          sourceState.replace(offset: offset, length: length, with: text)
+      do {
+        // Update initial state
+        _ = try document.update() { sourceState in
+          for case .replaceText(let offset, let length, let text) in priorActions {
+            sourceState.replace(offset: offset, length: length, with: text)
+          }
         }
+      } catch {
+        errors.append(error)
+        return errors
       }
     }
 
@@ -84,29 +100,38 @@ public class StressTester {
       if options.printActions {
         print(action)
       }
-      switch action {
-      case .cursorInfo(let offset):
-        try report(document.cursorInfo(offset: offset))
-      case .codeComplete(let offset, let expectedResult):
-        try report(document.codeComplete(offset: offset, expectedResult: expectedResult))
-      case .rangeInfo(let offset, let length):
-        try report(document.rangeInfo(offset: offset, length: length))
-      case .replaceText(let offset, let length, let text):
-        _ = try document.replaceText(offset: offset, length: length, text: text)
-      case .format(let offset):
-        try report(document.format(offset: offset))
-      case .typeContextInfo(let offset):
-        try report(document.typeContextInfo(offset: offset))
-      case .conformingMethodList(let offset):
-        try report(document.conformingMethodList(offset: offset, typeList: options.conformingMethodsTypeList))
-      case .collectExpressionType:
-        try report(document.collectExpressionType())
-      case .testModule:
-        try report(document.moduleInterfaceGen())
+      do {
+        switch action {
+        case .cursorInfo(let offset):
+          try report(document.cursorInfo(offset: offset))
+        case .codeComplete(let offset, let expectedResult):
+          try report(document.codeComplete(offset: offset, expectedResult: expectedResult))
+        case .rangeInfo(let offset, let length):
+          try report(document.rangeInfo(offset: offset, length: length))
+        case .replaceText(let offset, let length, let text):
+          _ = try document.replaceText(offset: offset, length: length, text: text)
+        case .format(let offset):
+          try report(document.format(offset: offset))
+        case .typeContextInfo(let offset):
+          try report(document.typeContextInfo(offset: offset))
+        case .conformingMethodList(let offset):
+          try report(document.conformingMethodList(offset: offset, typeList: options.conformingMethodsTypeList))
+        case .collectExpressionType:
+          try report(document.collectExpressionType())
+        case .testModule:
+          try report(document.moduleInterfaceGen())
+        }
+      } catch {
+        errors.append(error)
       }
     }
 
-    try document.close()
+    do {
+      try document.close()
+    } catch {
+      errors.append(error)
+    }
+    return errors
   }
 
   private func computeActions(from tree: SourceFileSyntax) -> (page: [Action], priorActions: [Action]) {

SourceKitStressTester/Sources/StressTester/StressTesterTool.swift

@@ -14,6 +14,7 @@ import ArgumentParser
 import Foundation
 import Common
 import SwiftSyntax
+import Dispatch
 
 public struct StressTesterTool: ParsableCommand {
   public static var configuration = CommandConfiguration(
@@ -159,20 +160,34 @@ public struct StressTesterTool: ParsableCommand {
         }
     )
 
-    do {
-      let processedArgs = CompilerArgs(for: file,
-                                       args: compilerArgs,
-                                       tempDir: tempDir!)
-      let tester = StressTester(options: options)
-      try tester.run(swiftc: swiftc!, compilerArgs: processedArgs)
-    } catch let error as SourceKitError {
-      let message = StressTesterMessage.detected(error)
-      try StressTesterTool.report(message, as: format)
-      throw ExitCode.failure
-    }
+    let processedArgs = CompilerArgs(for: file,
+                                        args: compilerArgs,
+                                        tempDir: tempDir!)
+    let tester = StressTester(options: options)
 
-    // Leave for debugging purposes if there was an error
-    try FileManager.default.removeItem(at: tempDir!)
+    DispatchQueue.global(qos: .userInitiated).async { [self] in
+      do {
+        let errors = tester.run(swiftc: swiftc!, compilerArgs: processedArgs)
+
+        if !errors.isEmpty {
+          for error in errors {
+            if let error = error as? SourceKitError {
+              let message = StressTesterMessage.detected(error)
+              try StressTesterTool.report(message, as: format)
+            } else {
+              throw error
+            }
+          }
+          throw ExitCode.failure
+        }
+
+        // Leave for debugging purposes if there was an error
+        try FileManager.default.removeItem(at: tempDir!)
+      } catch {
+        StressTesterTool.exit(withError: error)
+      }
+      StressTesterTool.exit(withError: nil)
+    }
   }
 
   private static func report<T>(_ message: T, as format: OutputFormat) throws

SourceKitStressTester/Sources/SwiftCWrapper/FailFastOperationQueue.swift

@@ -12,6 +12,8 @@
 
 import Foundation
 
+/// An operation queue that stops scheduling new operations as soon as the the
+/// completion handler returns `false`.
 public final class FailFastOperationQueue<Item: Operation> {
   private let serialQueue = DispatchQueue(label: "\(FailFastOperationQueue.self)")
   private let queue = OperationQueue()

SourceKitStressTester/Sources/SwiftCWrapper/SwiftCWrapper.swift

@@ -134,7 +134,11 @@ public struct SwiftCWrapper {
       progress?.update(step: completed, total: total, text: message)
       orderingHandler?.complete(operation.responses, at: index, setLast: !operation.status.isPassed)
       operation.responses.removeAll()
-      return operation.status.isPassed
+      // We can control whether to stop scheduling new operations here. As long
+      // as we return `true`, the stress tester continues to schedule new
+      // test operations. To stop at the first failure, return
+      // `operation.status.isPassed`.
+      return true
     }
     queue.waitUntilFinished()
 
@@ -151,29 +155,31 @@ public struct SwiftCWrapper {
 
     // Determine the set of processed files and the first failure (if any)
     var processedFiles = Set<String>()
-    var detectedIssue: StressTesterIssue? = nil
+    var detectedIssues: [StressTesterIssue] = []
     for operation in operations where detectedIssue == nil {
       switch operation.status {
       case .cancelled:
         fatalError("cancelled operation before failed operation")
       case .unexecuted:
         fatalError("unterminated operation")
       case .errored(let status):
-        detectedIssue = .errored(status: status, file: operation.file,
-                                 arguments: escapeArgs(operation.args))
+        detectedIssues.append(.errored(status: status, file: operation.file,
+                                 arguments: escapeArgs(operation.args)))
       case .failed(let sourceKitError):
-        detectedIssue = .failed(sourceKitError: sourceKitError,
-                                arguments: escapeArgs(operation.args))
+        detectedIssues.append(.failed(sourceKitError: sourceKitError,
+                                arguments: escapeArgs(operation.args)))
         fallthrough
       case .passed:
         processedFiles.insert(operation.file)
       }
     }
 
     let matchingSpec = try issueManager?.update(for: processedFiles, issue: detectedIssue)
-    try report(detectedIssue, matching: matchingSpec)
+    for detectedIssue in detectedIssues {
+      try report(detectedIssue, matching: matchingSpec)
+    }
 
-    if detectedIssue == nil || matchingSpec != nil {
+    if !detectedIssue.isEmpty || matchingSpec != nil {
       return EXIT_SUCCESS
     }
     return ignoreIssues ? swiftcResult.status : EXIT_FAILURE

SourceKitStressTester/Sources/SwiftSourceKit/SourcekitdClient.swift

@@ -13,31 +13,111 @@
 //===----------------------------------------------------------------------===//
 
 import sourcekitd
+import Dispatch
+
+fileprivate class Object {}
 
 public class SourceKitdService {
 
+  enum State {
+    case running
+    case interrupted
+    case semaDisabled
+  }
+
+  /// The queue that makes sure only one request is executed at a time
+  private let requestQueue = DispatchQueue(label: "SourceKitdService.requestQueue", qos: .userInitiated)
+  /// The queue that guards access to the `state` and `stateChangeHandlers` variables
+  private let stateQueue = DispatchQueue(label: "SourceKitdService.stateQueue", qos: .userInitiated)
+  private var state: State = .running {
+    didSet {
+      if #available(macOS 10.12, *) {
+        dispatchPrecondition(condition: .onQueue(stateQueue))
+      }
+      for handler in stateChangeHandlers.values {
+        handler()
+      }
+    }
+  }
+  private var stateChangeHandlers: [ObjectIdentifier: () -> Void] = [:]
+
   public init() {
+    sourcekitd_set_notification_handler { [self] resp in
+      stateQueue.async {
+        let response = SourceKitdResponse(resp: resp!)
+
+        if self.state == .interrupted {
+          self.state = .semaDisabled
+
+          // sourcekitd came back online. Poke it to restore semantic functionality
+          let request = SourceKitdRequest(uid: .request_CursorInfo)
+          request.addParameter(.key_SourceText, value: "")
+          _ = sourcekitd_send_request_sync(request.rawRequest)
+        }
+
+        if response.isConnectionInterruptionError {
+          self.state = .interrupted
+        } else if response.isSemaDisabledNotification {
+          self.state = .semaDisabled
+        } else if response.isSemaEnabledNotification {
+          self.state = .running
+        }
+      }
+    }
     sourcekitd_initialize()
   }
   deinit {
     sourcekitd_shutdown()
   }
 
+  /// Execute `callback` one this service is in `desiredState`
+  private func waitForState(_ desiredState: State, callback: @escaping () -> Void) {
+    stateQueue.async { [self] in
+      if state == desiredState {
+        callback()
+      } else {
+        let identifier = ObjectIdentifier(Object())
+        stateChangeHandlers[identifier] = { [self] in
+          if #available(macOS 10.12, *) {
+            dispatchPrecondition(condition: .onQueue(stateQueue))
+          }
+          if state == desiredState {
+            callback()
+            stateChangeHandlers[identifier] = nil
+          }
+        }
+      }
+    }
+  }
+
+  /// Block the current thread until this service is in `desiredState`.
+  private func blockUntilState(_ desiredState: State) {
+    let semaphore = DispatchSemaphore(value: 0)
+    waitForState(desiredState) {
+      semaphore.signal()
+    }
+    semaphore.wait()
+  }
+
   /// Send a request synchronously with a handler for its response.
   /// - Parameter request: The request to send.
   /// - Returns: The response from the sourcekitd service.
   public func sendSyn(request: SourceKitdRequest) -> SourceKitdResponse {
-    return SourceKitdResponse(resp: sourcekitd_send_request_sync(request.rawRequest))
+    return requestQueue.sync {
+      blockUntilState(.running)
+      return SourceKitdResponse(resp: sourcekitd_send_request_sync(request.rawRequest))
+    }
   }
 
   /// Send a request asynchronously with a handler for its response.
   /// - Parameter request: The request to send.
   /// - Parameter handler: The handler for the response in the future.
   public func send(request: SourceKitdRequest,
                    handler: @escaping (SourceKitdResponse) -> ())  {
-    sourcekitd_send_request(request.rawRequest, nil) { response in
-      guard let response = response else { return }
-      handler(SourceKitdResponse(resp: response))
+    requestQueue.async { [self] in
+      blockUntilState(.running)
+      let response = SourceKitdResponse(resp: sourcekitd_send_request_sync(request.rawRequest))
+      handler(response)
     }
   }
 }

SourceKitStressTester/Sources/SwiftSourceKit/SourcekitdResponse.swift

@@ -213,6 +213,10 @@ public class SourceKitdResponse: CustomStringConvertible {
 
   private let resp: sourcekitd_response_t
 
+  public var isNull: Bool {
+    return sourcekitd_variant_get_type(sourcekitd_response_get_value(resp)).rawValue == SOURCEKITD_VARIANT_TYPE_NULL.rawValue
+  }
+
   public var value: Dictionary {
     return Dictionary(dict: sourcekitd_response_get_value(resp), context: self)
   }
@@ -234,6 +238,20 @@ public class SourceKitdResponse: CustomStringConvertible {
     return value.getOptional(.key_Notification) != nil
   }
 
+  public var isSemaDisabledNotification: Bool {
+    if isNull { return false }
+    guard let notification = value.getOptional(.key_Notification)?.getUID()
+    else { return false }
+    return notification == .semaDisabledNotification
+  }
+
+  public var isSemaEnabledNotification: Bool {
+    if isNull { return false }
+    guard let notification = value.getOptional(.key_Notification)?.getUID()
+    else { return false }
+    return notification == .semaEnabledNotification
+  }
+
   /// Whether or not this response represents a connection interruption error.
   public var isConnectionInterruptionError: Bool {
     return sourcekitd_response_is_error(resp) &&

SourceKitStressTester/Sources/SwiftSourceKit/UIDs.swift

@@ -376,5 +376,9 @@ extension SourceKitdUID {
   public static let kind_SyntaxTreeSerializationByteTree = SourceKitdUID(string: "source.syntaxtree.serialization.format.bytetree")
 
   public static let compilerCrashedNotification = SourceKitdUID(string: "notification.toolchain-compiler-crashed")
+  public static let semaDisabledNotification = SourceKitdUID(string:
+      "source.notification.sema_disabled")
+  public static let semaEnabledNotification = SourceKitdUID(string:
+      "source.notification.sema_enabled")
   public static let source_notification_editor_documentupdate = SourceKitdUID(string: "source.notification.editor.documentupdate")
 }

SourceKitStressTester/Sources/sk-stress-test/main.swift

@@ -11,5 +11,9 @@
 //===----------------------------------------------------------------------===//
 
 import StressTester
+import Dispatch
 
-StressTesterTool.main()
+DispatchQueue.main.async {
+  StressTesterTool.main()
+}
+dispatchMain()