diff --git a/.gitignore b/.gitignore index b13934290..32c80ecff 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ default.profraw Package.resolved /.build +/.index-build /Packages /*.xcodeproj /*.sublime-project diff --git a/Package.swift b/Package.swift index b93f1079f..0f8e29557 100644 --- a/Package.swift +++ b/Package.swift @@ -173,6 +173,8 @@ let package = Package( .target( name: "SemanticIndex", dependencies: [ + "CAtomics", + "LanguageServerProtocol", "LSPLogging", "SKCore", .product(name: "IndexStoreDB", package: "indexstore-db"), diff --git a/Sources/LSPLogging/Logging.swift b/Sources/LSPLogging/Logging.swift index 079924750..35437e71e 100644 --- a/Sources/LSPLogging/Logging.swift +++ b/Sources/LSPLogging/Logging.swift @@ -13,7 +13,7 @@ /// Which log level to use (from https://developer.apple.com/wwdc20/10168?time=604) /// - Debug: Useful only during debugging (only logged during debugging) /// - Info: Helpful but not essential for troubleshooting (not persisted, logged to memory) -/// - Notice/log (Default): Essential for troubleshooting +/// - Notice/log/default: Essential for troubleshooting /// - Error: Error seen during execution /// - Used eg. if the user sends an erroneous request or if a request fails /// - Fault: Bug in program diff --git a/Sources/LSPLogging/NonDarwinLogging.swift b/Sources/LSPLogging/NonDarwinLogging.swift index f89d568b9..332e3e7c9 100644 --- a/Sources/LSPLogging/NonDarwinLogging.swift +++ b/Sources/LSPLogging/NonDarwinLogging.swift @@ -330,7 +330,7 @@ public struct NonDarwinLogger: Sendable { log(level: .info, message) } - /// Log a message at the `log` level. + /// Log a message at the `default` level. public func log(_ message: NonDarwinLogMessage) { log(level: .default, message) } diff --git a/Sources/SKCore/BuildSystemManager.swift b/Sources/SKCore/BuildSystemManager.swift index ea726f942..a7100f9ac 100644 --- a/Sources/SKCore/BuildSystemManager.swift +++ b/Sources/SKCore/BuildSystemManager.swift @@ -159,7 +159,8 @@ extension BuildSystemManager { /// references to that C file in the build settings by the header file. public func buildSettingsInferredFromMainFile( for document: DocumentURI, - language: Language + language: Language, + logBuildSettings: Bool = true ) async -> FileBuildSettings? { let mainFile = await mainFile(for: document, language: language) guard var settings = await buildSettings(for: mainFile, language: language) else { @@ -170,7 +171,9 @@ extension BuildSystemManager { // to reference `document` instead of `mainFile`. settings = settings.patching(newFile: document.pseudoPath, originalFile: mainFile.pseudoPath) } - await BuildSettingsLogger.shared.log(settings: settings, for: document) + if logBuildSettings { + await BuildSettingsLogger.shared.log(settings: settings, for: document) + } return settings } @@ -349,16 +352,24 @@ extension BuildSystemManager { // MARK: - Build settings logger /// Shared logger that only logs build settings for a file once unless they change -fileprivate actor BuildSettingsLogger { - static let shared = BuildSettingsLogger() +public actor BuildSettingsLogger { + public static let shared = BuildSettingsLogger() private var loggedSettings: [DocumentURI: FileBuildSettings] = [:] - func log(settings: FileBuildSettings, for uri: DocumentURI) { + public func log(level: LogLevel = .default, settings: FileBuildSettings, for uri: DocumentURI) { guard loggedSettings[uri] != settings else { return } loggedSettings[uri] = settings + Self.log(level: level, settings: settings, for: uri) + } + + /// Log the given build settings. + /// + /// In contrast to the instance method `log`, this will always log the build settings. The instance method only logs + /// the build settings if they have changed. + public static func log(level: LogLevel = .default, settings: FileBuildSettings, for uri: DocumentURI) { let log = """ Compiler Arguments: \(settings.compilerArguments.joined(separator: "\n")) @@ -370,6 +381,7 @@ fileprivate actor BuildSettingsLogger { let chunks = splitLongMultilineMessage(message: log) for (index, chunk) in chunks.enumerated() { logger.log( + level: level, """ Build settings for \(uri.forLogging) (\(index + 1)/\(chunks.count)) \(chunk) diff --git a/Sources/SKSupport/CMakeLists.txt b/Sources/SKSupport/CMakeLists.txt index 5453bb59d..7cb96d058 100644 --- a/Sources/SKSupport/CMakeLists.txt +++ b/Sources/SKSupport/CMakeLists.txt @@ -14,6 +14,7 @@ add_library(SKSupport STATIC Process+WaitUntilExitWithCancellation.swift Random.swift Result.swift + SwitchableProcessResultExitStatus.swift ThreadSafeBox.swift WorkspaceType.swift ) diff --git a/Sources/SKSupport/SwitchableProcessResultExitStatus.swift b/Sources/SKSupport/SwitchableProcessResultExitStatus.swift new file mode 100644 index 000000000..8e6f85733 --- /dev/null +++ b/Sources/SKSupport/SwitchableProcessResultExitStatus.swift @@ -0,0 +1,45 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +// We need to import all of TSCBasic because otherwise we can't refer to Process.ExitStatus (rdar://127577691) +import struct TSCBasic.ProcessResult + +/// Same as `ProcessResult.ExitStatus` in tools-support-core but has the same cases on all platforms and is thus easier +/// to switch over +public enum SwitchableProcessResultExitStatus { + /// The process was terminated normally with a exit code. + case terminated(code: Int32) + /// The process was terminated abnormally. + case abnormal(exception: UInt32) + /// The process was terminated due to a signal. + case signalled(signal: Int32) +} + +extension ProcessResult.ExitStatus { + public var exhaustivelySwitchable: SwitchableProcessResultExitStatus { + #if os(Windows) + switch self { + case .terminated(let code): + return .terminated(code: code) + case .abnormal(let exception): + return .abnormal(exception: exception) + } + #else + switch self { + case .terminated(let code): + return .terminated(code: code) + case .signalled(let signal): + return .signalled(signal: signal) + } + #endif + } +} diff --git a/Sources/SKTestSupport/SwiftPMTestProject.swift b/Sources/SKTestSupport/SwiftPMTestProject.swift index aa2737cea..72334c0d6 100644 --- a/Sources/SKTestSupport/SwiftPMTestProject.swift +++ b/Sources/SKTestSupport/SwiftPMTestProject.swift @@ -43,6 +43,7 @@ public class SwiftPMTestProject: MultiFileTestProject { build: Bool = false, allowBuildFailure: Bool = false, serverOptions: SourceKitLSPServer.Options = .testDefault, + pollIndex: Bool = true, usePullDiagnostics: Bool = true, testName: String = #function ) async throws { @@ -77,8 +78,10 @@ public class SwiftPMTestProject: MultiFileTestProject { try await Self.build(at: self.scratchDirectory) } } - // Wait for the indexstore-db to finish indexing - _ = try await testClient.send(PollIndexRequest()) + if pollIndex { + // Wait for the indexstore-db to finish indexing + _ = try await testClient.send(PollIndexRequest()) + } } /// Build a SwiftPM package package manifest is located in the directory at `path`. diff --git a/Sources/SKTestSupport/WrappedSemaphore.swift b/Sources/SKTestSupport/WrappedSemaphore.swift new file mode 100644 index 000000000..ee2036557 --- /dev/null +++ b/Sources/SKTestSupport/WrappedSemaphore.swift @@ -0,0 +1,36 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Dispatch + +/// Wrapper around `DispatchSemaphore` so that Swift Concurrency doesn't complain about the usage of semaphores in the +/// tests. +/// +/// This should only be used for tests that test priority escalation and thus cannot await a `Task` (which would cause +/// priority elevations). +public struct WrappedSemaphore { + let semaphore = DispatchSemaphore(value: 0) + + public init() {} + + public func signal(value: Int = 1) { + for _ in 0..) +} + +/// Schedules index tasks and keeps track of the index status of files. +public final actor SemanticIndexManager { + /// The underlying index. This is used to check if the index of a file is already up-to-date, in which case it doesn't + /// need to be indexed again. + private let index: CheckedIndex + + /// The build system manager that is used to get compiler arguments for a file. + private let buildSystemManager: BuildSystemManager + + /// The index status of the source files that the `SemanticIndexManager` knows about. + /// + /// Files that have never been indexed are not in this dictionary. + private var indexStatus: [DocumentURI: FileIndexStatus] = [:] + + /// The `TaskScheduler` that manages the scheduling of index tasks. This is shared among all `SemanticIndexManager`s + /// in the process, to ensure that we don't schedule more index operations than processor cores from multiple + /// workspaces. + private let indexTaskScheduler: TaskScheduler + + /// Callback that is called when an index task has finished. + /// + /// Currently only used for testing. + private let indexTaskDidFinish: (@Sendable (UpdateIndexStoreTaskDescription) -> Void)? + + // MARK: - Public API + + public init( + index: UncheckedIndex, + buildSystemManager: BuildSystemManager, + indexTaskScheduler: TaskScheduler, + indexTaskDidFinish: (@Sendable (UpdateIndexStoreTaskDescription) -> Void)? + ) { + self.index = index.checked(for: .modifiedFiles) + self.buildSystemManager = buildSystemManager + self.indexTaskScheduler = indexTaskScheduler + self.indexTaskDidFinish = indexTaskDidFinish + } + + /// Schedules a task to index all files in `files` that don't already have an up-to-date index. + /// Returns immediately after scheduling that task. + /// + /// Indexing is being performed with a low priority. + public func scheduleBackgroundIndex(files: some Collection) { + self.index(files: files, priority: .low) + } + + /// Wait for all in-progress index tasks to finish. + public func waitForUpToDateIndex() async { + logger.info("Waiting for up-to-date index") + await withTaskGroup(of: Void.self) { taskGroup in + for (_, status) in indexStatus { + switch status { + case .inProgress(let task): + taskGroup.addTask { + await task.value + } + case .upToDate: + break + } + } + await taskGroup.waitForAll() + } + index.pollForUnitChangesAndWait() + logger.debug("Done waiting for up-to-date index") + } + + /// Ensure that the index for the given files is up-to-date. + /// + /// This tries to produce an up-to-date index for the given files as quickly as possible. To achieve this, it might + /// suspend previous target-wide index tasks in favor of index tasks that index a fewer files. + public func waitForUpToDateIndex(for uris: some Collection) async { + logger.info( + "Waiting for up-to-date index for \(uris.map { $0.fileURL?.lastPathComponent ?? $0.stringValue }.joined(separator: ", "))" + ) + let filesWithOutOfDateIndex = uris.filter { uri in + switch indexStatus[uri] { + case .inProgress, nil: return true + case .upToDate: return false + } + } + // Create a new index task for the files that aren't up-to-date. The newly scheduled index tasks will + // - Wait for the existing index operations to finish if they have the same number of files. + // - Reschedule the background index task in favor of an index task with fewer source files. + await self.index(files: filesWithOutOfDateIndex, priority: nil).value + index.pollForUnitChangesAndWait() + logger.debug("Done waiting for up-to-date index") + } + + // MARK: - Helper functions + + /// Index the given set of files at the given priority. + /// + /// The returned task finishes when all files are indexed. + @discardableResult + private func index(files: some Collection, priority: TaskPriority?) -> Task { + let outOfDateFiles = files.filter { + if case .upToDate = indexStatus[$0] { + return false + } + return true + } + + var indexTasks: [Task] = [] + + // TODO (indexing): Group index operations by target when we support background preparation. + for files in outOfDateFiles.partition(intoNumberOfBatches: ProcessInfo.processInfo.processorCount * 5) { + let indexTask = Task(priority: priority) { + await self.indexTaskScheduler.schedule( + priority: priority, + UpdateIndexStoreTaskDescription( + filesToIndex: Set(files), + buildSystemManager: self.buildSystemManager, + index: self.index, + didFinishCallback: { [weak self] taskDescription in + self?.indexTaskDidFinish?(taskDescription) + } + ) + ).value + for file in files { + indexStatus[file] = .upToDate + } + } + indexTasks.append(indexTask) + + for file in files { + indexStatus[file] = .inProgress(indexTask) + } + } + let indexTasksImmutable = indexTasks + + return Task(priority: priority) { + await withTaskGroup(of: Void.self) { taskGroup in + for indexTask in indexTasksImmutable { + taskGroup.addTask(priority: priority) { + await indexTask.value + } + } + await taskGroup.waitForAll() + } + } + } +} diff --git a/Sources/SemanticIndex/UpdateIndexStoreTaskDescription.swift b/Sources/SemanticIndex/UpdateIndexStoreTaskDescription.swift new file mode 100644 index 000000000..9ef7924a6 --- /dev/null +++ b/Sources/SemanticIndex/UpdateIndexStoreTaskDescription.swift @@ -0,0 +1,311 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import CAtomics +import Foundation +import LSPLogging +import LanguageServerProtocol +import SKCore +import SKSupport + +import struct TSCBasic.AbsolutePath +import class TSCBasic.Process + +private var updateIndexStoreIDForLogging = AtomicUInt32(initialValue: 1) + +/// Describes a task to index a set of source files. +/// +/// This task description can be scheduled in a `TaskScheduler`. +public struct UpdateIndexStoreTaskDescription: TaskDescriptionProtocol { + public let id = updateIndexStoreIDForLogging.fetchAndIncrement() + + /// The files that should be indexed. + private let filesToIndex: Set + + /// The build system manager that is used to get the toolchain and build settings for the files to index. + private let buildSystemManager: BuildSystemManager + + /// A reference to the underlying index store. Used to check if the index is already up-to-date for a file, in which + /// case we don't need to index it again. + private let index: CheckedIndex + + /// A callback that is called when the index task finishes + private let didFinishCallback: @Sendable (UpdateIndexStoreTaskDescription) -> Void + + /// The task is idempotent because indexing the same file twice produces the same result as indexing it once. + public var isIdempotent: Bool { true } + + public var estimatedCPUCoreCount: Int { 1 } + + public var description: String { + return self.redactedDescription + } + + public var redactedDescription: String { + return "indexing-\(id)" + } + + init( + filesToIndex: Set, + buildSystemManager: BuildSystemManager, + index: CheckedIndex, + didFinishCallback: @escaping @Sendable (UpdateIndexStoreTaskDescription) -> Void + ) { + self.filesToIndex = filesToIndex + self.buildSystemManager = buildSystemManager + self.index = index + self.didFinishCallback = didFinishCallback + } + + public func execute() async { + defer { + didFinishCallback(self) + } + // Only use the last two digits of the indexing ID for the logging scope to avoid creating too many scopes. + // See comment in `withLoggingScope`. + // The last 2 digits should be sufficient to differentiate between multiple concurrently running indexing operation. + await withLoggingSubsystemAndScope( + subsystem: "org.swift.sourcekit-lsp.indexing", + scope: "update-indexstore-\(id % 100)" + ) { + let startDate = Date() + + let filesToIndexDescription = filesToIndex.map { $0.fileURL?.lastPathComponent ?? $0.stringValue } + .joined(separator: ", ") + logger.log( + "Starting updating index store with priority \(Task.currentPriority.rawValue, privacy: .public): \(filesToIndexDescription)" + ) + let filesToIndex = filesToIndex.sorted(by: { $0.stringValue < $1.stringValue }) + // TODO (indexing): Once swiftc supports it, we should group files by target and index files within the same + // target together in one swiftc invocation. + for file in filesToIndex { + await updateIndexStoreForSingleFile(file) + } + logger.log( + "Finished updating index store in \(Date().timeIntervalSince(startDate) * 1000, privacy: .public)ms: \(filesToIndexDescription)" + ) + } + } + + public func dependencies( + to currentlyExecutingTasks: [UpdateIndexStoreTaskDescription] + ) -> [TaskDependencyAction] { + return currentlyExecutingTasks.compactMap { (other) -> TaskDependencyAction? in + guard !other.filesToIndex.intersection(filesToIndex).isEmpty else { + // Disjoint sets of files can be indexed concurrently. + return nil + } + if self.filesToIndex.count < other.filesToIndex.count { + // If there is an index operation with more files already running, suspend it. + // The most common use case for this is if we schedule an entire target to be indexed in the background and then + // need a single file indexed for use interaction. We should suspend the target-wide indexing and just index + // the current file to get index data for it ASAP. + return .cancelAndRescheduleDependency(other) + } else { + return .waitAndElevatePriorityOfDependency(other) + } + } + } + + private func updateIndexStoreForSingleFile(_ uri: DocumentURI) async { + guard let url = uri.fileURL else { + // The URI is not a file, so there's nothing we can index. + return + } + guard !index.hasUpToDateUnit(for: url) else { + // We consider a file's index up-to-date if we have any up-to-date unit. Changing build settings does not + // invalidate the up-to-date status of the index. + return + } + guard let language = await buildSystemManager.defaultLanguage(for: uri) else { + logger.error("Not indexing \(uri.forLogging) because its language could not be determined") + return + } + let buildSettings = await buildSystemManager.buildSettingsInferredFromMainFile( + for: uri, + language: language, + logBuildSettings: false + ) + guard let buildSettings else { + logger.error("Not indexing \(uri.forLogging) because it has no compiler arguments") + return + } + guard !buildSettings.isFallback else { + // Only index with real build settings. Indexing with fallback arguments could result in worse results than not + // indexing at all: If a file has been indexed with real build settings before, had a tiny modification made but + // we don't have any real build settings when it should get re-indexed. Then it's better to have the stale index + // from correct compiler arguments than no index at all. + logger.error("Not updating index store for \(uri.forLogging) because it has fallback compiler arguments") + return + } + guard let toolchain = await buildSystemManager.toolchain(for: uri, language) else { + logger.error( + "Not updating index store for \(uri.forLogging) because no toolchain could be determined for the document" + ) + return + } + switch language { + case .swift: + do { + try await updateIndexStore(forSwiftFile: uri, buildSettings: buildSettings, toolchain: toolchain) + } catch { + logger.error("Updating index store for \(uri) failed: \(error.forLogging)") + BuildSettingsLogger.log(settings: buildSettings, for: uri) + } + case .c, .cpp, .objective_c, .objective_cpp: + // TODO (indexing): Support indexing of clang files, including headers. + break + default: + logger.error( + "Not updating index store for \(uri) because it is a language that is not supported by background indexing" + ) + } + } + + private func updateIndexStore( + forSwiftFile uri: DocumentURI, + buildSettings: FileBuildSettings, + toolchain: Toolchain + ) async throws { + let indexingArguments = adjustSwiftCompilerArgumentsForIndexStoreUpdate( + buildSettings.compilerArguments, + fileToIndex: uri + ) + + guard let swiftc = toolchain.swiftc else { + logger.error( + "Not updating index store for \(uri.forLogging) because toolchain \(toolchain.identifier) does not contain a Swift compiler" + ) + return + } + + let process = + if let workingDirectory = buildSettings.workingDirectory { + Process( + arguments: [swiftc.pathString] + indexingArguments, + workingDirectory: try AbsolutePath(validating: workingDirectory) + ) + } else { + Process(arguments: [swiftc.pathString] + indexingArguments) + } + try process.launch() + let result = try await process.waitUntilExitSendingSigIntOnTaskCancellation() + switch result.exitStatus.exhaustivelySwitchable { + case .terminated(code: 0): + break + case .terminated(code: let code): + // This most likely happens if there are compilation errors in the source file. This is nothing to worry about. + let stdout = (try? String(bytes: result.output.get(), encoding: .utf8)) ?? "" + let stderr = (try? String(bytes: result.stderrOutput.get(), encoding: .utf8)) ?? "" + // Indexing will frequently fail if the source code is in an invalid state. Thus, log the failure at a low level. + logger.debug( + """ + Updating index store for Swift file \(uri.forLogging) terminated with non-zero exit code \(code) + Stderr: + \(stderr) + Stdout: + \(stdout) + """ + ) + BuildSettingsLogger.log(level: .debug, settings: buildSettings, for: uri) + case .signalled(signal: let signal): + if !Task.isCancelled { + // The indexing job finished with a signal. Could be because the compiler crashed. + // Ignore signal exit codes if this task has been cancelled because the compiler exits with SIGINT if it gets + // interrupted. + logger.error("Updating index store for Swift file \(uri.forLogging) signaled \(signal)") + BuildSettingsLogger.log(level: .error, settings: buildSettings, for: uri) + } + case .abnormal(exception: let exception): + if !Task.isCancelled { + logger.error("Updating index store for Swift file \(uri.forLogging) exited abnormally \(exception)") + BuildSettingsLogger.log(level: .error, settings: buildSettings, for: uri) + } + } + } +} + +/// Adjust compiler arguments that were created for building to compiler arguments that should be used for indexing. +/// +/// This removes compiler arguments that produce output files and adds arguments to index the file. +private func adjustSwiftCompilerArgumentsForIndexStoreUpdate( + _ compilerArguments: [String], + fileToIndex: DocumentURI +) -> [String] { + let removeFlags: Set = [ + "-c", + "-disable-cmo", + "-emit-dependencies", + "-emit-module-interface", + "-emit-module", + "-emit-module", + "-emit-objc-header", + "-incremental", + "-no-color-diagnostics", + "-parseable-output", + "-save-temps", + "-serialize-diagnostics", + "-use-frontend-parseable-output", + "-validate-clang-modules-once", + "-whole-module-optimization", + ] + + let removeArguments: Set = [ + "-clang-build-session-file", + "-emit-module-interface-path", + "-emit-module-path", + "-emit-objc-header-path", + "-emit-package-module-interface-path", + "-emit-private-module-interface-path", + "-num-threads", + "-o", + "-output-file-map", + ] + + let removeFrontendFlags: Set = [ + "-experimental-skip-non-inlinable-function-bodies", + "-experimental-skip-all-function-bodies", + ] + + var result: [String] = [] + result.reserveCapacity(compilerArguments.count) + var iterator = compilerArguments.makeIterator() + while let argument = iterator.next() { + if removeFlags.contains(argument) { + continue + } + if removeArguments.contains(argument) { + _ = iterator.next() + continue + } + if argument == "-Xfrontend" { + if let nextArgument = iterator.next() { + if removeFrontendFlags.contains(nextArgument) { + continue + } + result += [argument, nextArgument] + continue + } + result.append(argument) + } + result.append(argument) + } + result += [ + "-index-file", + "-index-file-path", fileToIndex.pseudoPath, + // batch mode is not compatible with -index-file + "-disable-batch-mode", + // Fake an output path so that we get a different unit file for every Swift file we background index + "-o", fileToIndex.pseudoPath + ".o", + ] + return result +} diff --git a/Sources/SourceKitLSP/SourceKitLSPServer.swift b/Sources/SourceKitLSP/SourceKitLSPServer.swift index fce17f037..cf3bc2811 100644 --- a/Sources/SourceKitLSP/SourceKitLSPServer.swift +++ b/Sources/SourceKitLSP/SourceKitLSPServer.swift @@ -449,6 +449,12 @@ public actor SourceKitLSPServer { let documentManager = DocumentManager() + /// The `TaskScheduler` that schedules all background indexing tasks. + /// + /// Shared process-wide to ensure the scheduled index operations across multiple workspaces don't exceed the maximum + /// number of processor cores that the user allocated to background indexing. + private let indexTaskScheduler: TaskScheduler + private var packageLoadingWorkDoneProgress = WorkDoneProgressState( "SourceKitLSP.SourceKitLSPServer.reloadPackage", title: "SourceKit-LSP: Reloading Package" @@ -519,6 +525,12 @@ public actor SourceKitLSPServer { self.onExit = onExit self.client = client + let processorCount = ProcessInfo.processInfo.processorCount + let lowPriorityCores = options.indexOptions.maxCoresPercentageToUseForBackgroundIndexing * Double(processorCount) + self.indexTaskScheduler = TaskScheduler(maxConcurrentTasksByPriority: [ + (TaskPriority.medium, processorCount), + (TaskPriority.low, max(Int(lowPriorityCores), 1)), + ]) } /// Search through all the parent directories of `uri` and check if any of these directories contain a workspace @@ -1152,6 +1164,7 @@ extension SourceKitLSPServer { options: options, compilationDatabaseSearchPaths: self.options.compilationDatabaseSearchPaths, indexOptions: self.options.indexOptions, + indexTaskScheduler: indexTaskScheduler, reloadPackageStatusCallback: { [weak self] status in guard let self else { return } guard capabilityRegistry.clientCapabilities.window?.workDoneProgress ?? false else { @@ -1220,7 +1233,8 @@ extension SourceKitLSPServer { options: self.options, underlyingBuildSystem: nil, index: nil, - indexDelegate: nil + indexDelegate: nil, + indexTaskScheduler: self.indexTaskScheduler ) self.workspacesAndIsImplicit.append((workspace: workspace, isImplicit: false)) @@ -2414,7 +2428,8 @@ extension SourceKitLSPServer { func pollIndex(_ req: PollIndexRequest) async throws -> VoidResponse { for workspace in workspaces { - workspace.uncheckedIndex?.underlyingIndexStoreDB.pollForUnitChangesAndWait() + await workspace.semanticIndexManager?.waitForUpToDateIndex() + workspace.index(checkedFor: .deletedFiles)?.pollForUnitChangesAndWait() } return VoidResponse() } diff --git a/Sources/SourceKitLSP/Workspace.swift b/Sources/SourceKitLSP/Workspace.swift index cb8d8ef52..5306b66f4 100644 --- a/Sources/SourceKitLSP/Workspace.swift +++ b/Sources/SourceKitLSP/Workspace.swift @@ -74,6 +74,12 @@ public final class Workspace { /// Language service for an open document, if available. var documentService: [DocumentURI: LanguageService] = [:] + /// The `SemanticIndexManager` that keeps track of whose file's index is up-to-date in the workspace and schedules + /// indexing and preparation tasks for files with out-of-date index. + /// + /// `nil` if background indexing is not enabled. + let semanticIndexManager: SemanticIndexManager? + public init( documentManager: DocumentManager, rootUri: DocumentURI?, @@ -82,7 +88,8 @@ public final class Workspace { options: SourceKitLSPServer.Options, underlyingBuildSystem: BuildSystem?, index uncheckedIndex: UncheckedIndex?, - indexDelegate: SourceKitIndexDelegate? + indexDelegate: SourceKitIndexDelegate?, + indexTaskScheduler: TaskScheduler ) async { self.documentManager = documentManager self.buildSetup = options.buildSetup @@ -95,6 +102,16 @@ public final class Workspace { mainFilesProvider: uncheckedIndex, toolchainRegistry: toolchainRegistry ) + if let uncheckedIndex, options.indexOptions.enableBackgroundIndexing { + self.semanticIndexManager = SemanticIndexManager( + index: uncheckedIndex, + buildSystemManager: buildSystemManager, + indexTaskScheduler: indexTaskScheduler, + indexTaskDidFinish: options.indexOptions.indexTaskDidFinish + ) + } else { + self.semanticIndexManager = nil + } await indexDelegate?.addMainFileChangedCallback { [weak self] in await self?.buildSystemManager.mainFilesChanged() } @@ -106,6 +123,9 @@ public final class Workspace { } // Trigger an initial population of `syntacticTestIndex`. await syntacticTestIndex.listOfTestFilesDidChange(buildSystemManager.testFiles()) + if let semanticIndexManager, let underlyingBuildSystem { + await semanticIndexManager.scheduleBackgroundIndex(files: await underlyingBuildSystem.sourceFiles().map(\.uri)) + } } /// Creates a workspace for a given root `URL`, inferring the `ExternalWorkspace` if possible. @@ -122,11 +142,16 @@ public final class Workspace { options: SourceKitLSPServer.Options, compilationDatabaseSearchPaths: [RelativePath], indexOptions: IndexOptions = IndexOptions(), + indexTaskScheduler: TaskScheduler, reloadPackageStatusCallback: @escaping (ReloadPackageStatus) async -> Void ) async throws { var buildSystem: BuildSystem? = nil if let rootUrl = rootUri.fileURL, let rootPath = try? AbsolutePath(validating: rootUrl.path) { + var options = options + if options.indexOptions.enableBackgroundIndexing, options.buildSetup.path == nil { + options.buildSetup.path = rootPath.appending(component: ".index-build") + } func createSwiftPMBuildSystem(rootUrl: URL) async -> SwiftPMBuildSystem? { return await SwiftPMBuildSystem( url: rootUrl, @@ -218,7 +243,8 @@ public final class Workspace { options: options, underlyingBuildSystem: buildSystem, index: UncheckedIndex(index), - indexDelegate: indexDelegate + indexDelegate: indexDelegate, + indexTaskScheduler: indexTaskScheduler ) } @@ -258,15 +284,34 @@ public struct IndexOptions { /// explicit calls to pollForUnitChangesAndWait(). public var listenToUnitEvents: Bool + /// Whether background indexing should be enabled. + public var enableBackgroundIndexing: Bool + + /// The percentage of the machine's cores that should at most be used for background indexing. + /// + /// Setting this to a value < 1 ensures that background indexing doesn't use all CPU resources. + public var maxCoresPercentageToUseForBackgroundIndexing: Double + + /// A callback that is called when an index task finishes. + /// + /// Intended for testing purposes. + public var indexTaskDidFinish: (@Sendable (UpdateIndexStoreTaskDescription) -> Void)? + public init( indexStorePath: AbsolutePath? = nil, indexDatabasePath: AbsolutePath? = nil, indexPrefixMappings: [PathPrefixMapping]? = nil, - listenToUnitEvents: Bool = true + listenToUnitEvents: Bool = true, + enableBackgroundIndexing: Bool = false, + maxCoresPercentageToUseForBackgroundIndexing: Double = 1, + indexTaskDidFinish: (@Sendable (UpdateIndexStoreTaskDescription) -> Void)? = nil ) { self.indexStorePath = indexStorePath self.indexDatabasePath = indexDatabasePath self.indexPrefixMappings = indexPrefixMappings self.listenToUnitEvents = listenToUnitEvents + self.enableBackgroundIndexing = enableBackgroundIndexing + self.maxCoresPercentageToUseForBackgroundIndexing = maxCoresPercentageToUseForBackgroundIndexing + self.indexTaskDidFinish = indexTaskDidFinish } } diff --git a/Tests/SKCoreTests/TaskSchedulerTests.swift b/Tests/SKCoreTests/TaskSchedulerTests.swift index db81c320b..9ca7a1bd0 100644 --- a/Tests/SKCoreTests/TaskSchedulerTests.swift +++ b/Tests/SKCoreTests/TaskSchedulerTests.swift @@ -11,6 +11,7 @@ //===----------------------------------------------------------------------===// import SKCore +import SKTestSupport import XCTest final class TaskSchedulerTests: XCTestCase { @@ -285,24 +286,6 @@ fileprivate actor TaskExecutionRecorder { } } -/// Wrapper around `DispatchSemaphore` so that Swift Concurrency doesn't complain about the usage of semaphores in the -/// tests. -fileprivate struct UnsafeSemaphore { - let semaphore = DispatchSemaphore(value: 0) - - func signal(value: Int = 1) { - for _ in 0..