Merge main into release/6.0

Documentation/Background Indexing.md → Documentation/Enable Experimental Background Indexing.md

@@ -22,9 +22,10 @@ Next, point your editor to use the just-built copy of SourceKit-LSP and enable b
 "swift.sourcekit-lsp.serverArguments": [ "--experimental-feature", "background-indexing" ],
 ```
 
+Background indexing requires a Swift 6 toolchain. You can download Swift 6 nightly toolchains from https://www.swift.org/download/#swift-60-development.
+
 ## Known issues
 
-- The only supported toolchain for background indexing are currently [Swift 6.0 nightly toolchain snapshots](https://www.swift.org/download/#swift-60-development). Older toolchains are not supported and the nightly toolchains from `main` are having issues because building a target non-deterministically builds for tools or the destination [#1288](https://github.com/apple/sourcekit-lsp/pull/1288#issuecomment-2111400459) [rdar://128100158](rdar://128100158)
 - Not really a background indexing related issue but Swift nightly toolchain snapshots are crashing on macOS 14.4 and 14.5 (swift#73327)[https://github.com/apple/swift/issues/73327]
   - Workaround: Run the toolchains on an older version of macOS, if possible
 - Background Indexing is only supported for SwiftPM projects [#1269](https://github.com/apple/sourcekit-lsp/issues/1269), [#1271](https://github.com/apple/sourcekit-lsp/issues/1271)

Sources/Diagnose/DiagnoseCommand.swift

@@ -407,8 +407,7 @@ public struct DiagnoseCommand: AsyncParsableCommand {
     // is responsible for showing the diagnose bundle location to the user
     if self.bundleOutputPath == nil {
       do {
-        let process = try Process.launch(arguments: ["open", "-R", bundlePath.path], workingDirectory: nil)
-        try await process.waitUntilExitSendingSigIntOnTaskCancellation()
+        _ = try await Process.run(arguments: ["open", "-R", bundlePath.path], workingDirectory: nil)
       } catch {
         // If revealing the bundle in Finder should fail, we don't care. We still printed the bundle path to stdout.
       }

Sources/Diagnose/IndexCommand.swift

@@ -68,13 +68,23 @@ public struct IndexCommand: AsyncParsableCommand {
   )
   var toolchainOverride: String?
 
+  @Option(
+    name: .customLong("experimental-index-feature"),
+    help: """
+      Enable an experimental sourcekit-lsp feature.
+      Available features are: \(ExperimentalFeature.allCases.map(\.rawValue).joined(separator: ", "))
+      """
+  )
+  var experimentalFeatures: [ExperimentalFeature] = []
+
   @Option(help: "The path to the project that should be indexed")
   var project: String
 
   public init() {}
 
   public func run() async throws {
     var serverOptions = SourceKitLSPServer.Options()
+    serverOptions.experimentalFeatures = Set(experimentalFeatures)
     serverOptions.experimentalFeatures.insert(.backgroundIndexing)
 
     let installPath =
@@ -109,3 +119,9 @@ fileprivate extension SourceKitLSPServer {
     }
   }
 }
+
+#if compiler(>=6)
+extension ExperimentalFeature: @retroactive ExpressibleByArgument {}
+#else
+extension ExperimentalFeature: ExpressibleByArgument {}
+#endif

Sources/Diagnose/MergeSwiftFiles.swift

@@ -27,10 +27,11 @@ extension RequestInfo {
 
     progressUpdate(0, "Merging all .swift files into a single file")
 
+    let compilerArgs = compilerArgs.filter { $0 != "-primary-file" && !$0.hasSuffix(".swift") } + ["$FILE"]
     let mergedRequestInfo = RequestInfo(
       requestTemplate: requestTemplate,
       offset: offset,
-      compilerArgs: compilerArgs.filter { !$0.hasSuffix(".swift") } + ["$FILE"],
+      compilerArgs: compilerArgs,
       fileContents: mergedFile
     )
 

Sources/Diagnose/ReduceFrontendCommand.swift

@@ -96,17 +96,18 @@ public struct ReduceFrontendCommand: AsyncParsableCommand {
       reproducerPredicate: nsPredicate
     )
 
+    defer {
+      progressBar.complete(success: true)
+    }
     let reducedRequestInfo = try await reduceFrontendIssue(
       frontendArgs: frontendArgs,
       using: executor
     ) { progress, message in
       progressBar.update(step: Int(progress * 100), total: 100, text: message)
     }
 
-    progressBar.complete(success: true)
-
     print("Reduced compiler arguments:")
-    print(reducedRequestInfo.compilerArgs)
+    print(reducedRequestInfo.compilerArgs.joined(separator: " "))
 
     print("")
     print("Reduced file contents:")

Sources/Diagnose/RequestInfo.swift

@@ -121,10 +121,12 @@ public struct RequestInfo: Sendable {
 
     // Inline the file list so we can reduce the compiler arguments by removing individual source files.
     // A couple `output-filelist`-related compiler arguments don't work with the file list inlined. Remove them as they
-    // are unlikely to be responsible for the swift-frontend cache
+    // are unlikely to be responsible for the swift-frontend cache.
+    // `-index-system-modules` is invalid when no output file lists are specified.
     while let frontendArg = iterator.next() {
       switch frontendArg {
-      case "-supplementary-output-file-map", "-output-filelist", "-index-unit-output-path-filelist":
+      case "-supplementary-output-file-map", "-output-filelist", "-index-unit-output-path-filelist",
+        "-index-system-modules":
         _ = iterator.next()
       case "-filelist":
         guard let fileList = iterator.next() else {

Sources/SKCore/BuildServerBuildSystem.swift

@@ -278,6 +278,10 @@ extension BuildServerBuildSystem: BuildSystem {
     return nil
   }
 
+  public func toolchain(for uri: DocumentURI, _ language: Language) async -> SKCore.Toolchain? {
+    return nil
+  }
+
   public func configuredTargets(for document: DocumentURI) async -> [ConfiguredTarget] {
     return [ConfiguredTarget(targetID: "dummy", runDestinationID: "dummy")]
   }

Sources/SKCore/BuildSystem.swift

@@ -179,6 +179,11 @@ public protocol BuildSystem: AnyObject, Sendable {
   /// If `nil` is returned, the language based on the file's extension.
   func defaultLanguage(for document: DocumentURI) async -> Language?
 
+  /// The toolchain that should be used to open the given document.
+  ///
+  /// If `nil` is returned, then the default toolchain for the given language is used.
+  func toolchain(for uri: DocumentURI, _ language: Language) async -> Toolchain?
+
   /// Register the given file for build-system level change notifications, such
   /// as command line flag changes, dependency changes, etc.
   ///

Sources/SKCore/BuildSystemManager.swift

@@ -104,9 +104,18 @@ extension BuildSystemManager {
 
   /// Returns the toolchain that should be used to process the given document.
   public func toolchain(for uri: DocumentURI, _ language: Language) async -> Toolchain? {
-    // To support multiple toolchains within a single workspace, we need to ask the build system which toolchain to use
-    // for this document.
-    return await toolchainRegistry.defaultToolchain(for: language)
+    if let toolchain = await buildSystem?.toolchain(for: uri, language) {
+      return toolchain
+    }
+
+    switch language {
+    case .swift:
+      return await toolchainRegistry.preferredToolchain(containing: [\.sourcekitd, \.swift, \.swiftc])
+    case .c, .cpp, .objective_c, .objective_cpp:
+      return await toolchainRegistry.preferredToolchain(containing: [\.clang, \.clangd])
+    default:
+      return nil
+    }
   }
 
   /// - Note: Needed so we can set the delegate from a different isolation context.

Sources/SKCore/CompilationDatabaseBuildSystem.swift

@@ -119,6 +119,10 @@ extension CompilationDatabaseBuildSystem: BuildSystem {
     return nil
   }
 
+  public func toolchain(for uri: DocumentURI, _ language: Language) async -> SKCore.Toolchain? {
+    return nil
+  }
+
   public func configuredTargets(for document: DocumentURI) async -> [ConfiguredTarget] {
     return [ConfiguredTarget(targetID: "dummy", runDestinationID: "dummy")]
   }

Sources/SKCore/IndexTaskID.swift

@@ -19,7 +19,7 @@ public enum IndexTaskID: Sendable {
   case updateIndexStore(id: UInt32)
 
   private static func numberToEmojis(_ number: Int, numEmojis: Int) -> String {
-    let emojis = ["🟥", "🟩", "🟦", "🟧", "⬜️", "🟪", "⬛️", "🟨", "🟫"]
+    let emojis = ["🟥", "🟩", "🟦", "⬜️", "🟪", "⬛️", "🟨", "🟫"]
     var number = abs(number)
     var result = ""
     for _ in 0..<numEmojis {
@@ -36,11 +36,12 @@ public enum IndexTaskID: Sendable {
   public var emojiRepresentation: String {
     // Multiply by 2 and optionally add 1 to make sure preparation and update index store have distinct IDs.
     // Run .hashValue to make sure we semi-randomly pick new emoji markers for new tasks
+    let numEmojis = 3
     switch self {
     case .preparation(id: let id):
-      return Self.numberToEmojis((id * 2).hashValue, numEmojis: 2)
+      return Self.numberToEmojis((id * 2).hashValue, numEmojis: numEmojis)
     case .updateIndexStore(id: let id):
-      return Self.numberToEmojis((id * 2 + 1).hashValue, numEmojis: 2)
+      return Self.numberToEmojis((id * 2 + 1).hashValue, numEmojis: numEmojis)
     }
   }
 }

Sources/SKCore/TaskScheduler.swift

@@ -541,3 +541,25 @@ fileprivate extension Collection<Int> {
     return result
   }
 }
+
+/// Version of the `withTaskPriorityChangedHandler` where the body doesn't throw.
+fileprivate func withTaskPriorityChangedHandler(
+  initialPriority: TaskPriority = Task.currentPriority,
+  pollingInterval: Duration = .seconds(0.1),
+  @_inheritActorContext operation: @escaping @Sendable () async -> Void,
+  taskPriorityChanged: @escaping @Sendable () -> Void
+) async {
+  do {
+    try await withTaskPriorityChangedHandler(
+      initialPriority: initialPriority,
+      pollingInterval: pollingInterval,
+      operation: operation as @Sendable () async throws -> Void,
+      taskPriorityChanged: taskPriorityChanged
+    )
+  } catch is CancellationError {
+  } catch {
+    // Since `operation` does not throw, the only error we expect `withTaskPriorityChangedHandler` to throw is a
+    // `CancellationError`, in which case we can just return.
+    logger.fault("Unexpected error thrown from withTaskPriorityChangedHandler: \(error.forLogging)")
+  }
+}

Sources/SKCore/ToolchainRegistry.swift

@@ -246,26 +246,14 @@ public final actor ToolchainRegistry {
     return darwinToolchainOverride ?? ToolchainRegistry.darwinDefaultToolchainIdentifier
   }
 
-  /// The toolchain to use for a document in the given language if the build system doesn't override it.
-  func defaultToolchain(for language: Language) -> Toolchain? {
-    let supportsLang = { (toolchain: Toolchain) -> Bool in
-      // FIXME: the fact that we're looking at clangd/sourcekitd instead of the compiler indicates this method needs a parameter stating what kind of tool we're looking for.
-      switch language {
-      case .swift:
-        return toolchain.sourcekitd != nil
-      case .c, .cpp, .objective_c, .objective_cpp:
-        return toolchain.clangd != nil
-      default:
-        return false
-      }
-    }
-
-    if let toolchain = self.default, supportsLang(toolchain) {
+  /// Returns the preferred toolchain that contains all the tools at the given key paths.
+  public func preferredToolchain(containing requiredTools: [KeyPath<Toolchain, AbsolutePath?>]) -> Toolchain? {
+    if let toolchain = self.default, requiredTools.allSatisfy({ toolchain[keyPath: $0] != nil }) {
       return toolchain
     }
 
     for toolchain in toolchains {
-      if supportsLang(toolchain) {
+      if requiredTools.allSatisfy({ toolchain[keyPath: $0] != nil }) {
         return toolchain
       }
     }

Sources/SKSupport/CMakeLists.txt

@@ -9,8 +9,7 @@ add_library(SKSupport STATIC
   FileSystem.swift
   LineTable.swift
   PipeAsStringHandler.swift
-  Process+LaunchWithWorkingDirectoryIfPossible.swift
-  Process+WaitUntilExitWithCancellation.swift
+  Process+Run.swift
   Random.swift
   Result.swift
   SwitchableProcessResultExitStatus.swift