From a88798129da9ad9e0c9b910e0ad1fa22cb0c8e61 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Tue, 21 May 2024 22:09:22 -0700 Subject: [PATCH 1/3] Add an option to show the files that are currently being index / targets being prepared in the work done progress --- .gitignore | 3 +- .../SemanticIndex/SemanticIndexManager.swift | 58 ++++++++++--------- .../SourceKitLSP/IndexProgressManager.swift | 58 ++++++++++++------- Sources/SourceKitLSP/SourceKitLSPServer.swift | 8 +-- Sources/SourceKitLSP/Workspace.swift | 20 +++++-- Sources/sourcekit-lsp/SourceKitLSP.swift | 13 ++++- .../BackgroundIndexingTests.swift | 2 +- .../SourceKitLSPTests/BuildSystemTests.swift | 2 +- 8 files changed, 101 insertions(+), 63 deletions(-) diff --git a/.gitignore b/.gitignore index 2000e2996..2abd0e14d 100644 --- a/.gitignore +++ b/.gitignore @@ -2,8 +2,7 @@ default.profraw Package.resolved /.build -/.index-build -/.linux-build +/.*-build /Packages /*.xcodeproj /*.sublime-project diff --git a/Sources/SemanticIndex/SemanticIndexManager.swift b/Sources/SemanticIndex/SemanticIndexManager.swift index 1500ab3b1..7a50c88e7 100644 --- a/Sources/SemanticIndex/SemanticIndexManager.swift +++ b/Sources/SemanticIndex/SemanticIndexManager.swift @@ -55,6 +55,12 @@ private enum InProgressIndexStore { case updatingIndexStore(updateIndexStoreTask: OpaqueQueuedIndexTask, indexTask: Task) } +/// Status of document indexing / target preparation in `inProgressIndexAndPreparationTasks`. +public enum IndexTaskStatus: Comparable { + case scheduled + case executing +} + /// Schedules index tasks and keeps track of the index status of files. public final actor SemanticIndexManager { /// The underlying index. This is used to check if the index of a file is already up-to-date, in which case it doesn't @@ -107,42 +113,36 @@ public final actor SemanticIndexManager { /// The parameter is the number of files that were scheduled to be indexed. private let indexTasksWereScheduled: @Sendable (_ numberOfFileScheduled: Int) -> Void - /// Callback that is called when an index task has finished. + /// Callback that is called when the progress status of an update indexstore or preparation task finishes. /// /// An object observing this property probably wants to check `inProgressIndexTasks` when the callback is called to /// get the current list of in-progress index tasks. /// - /// The number of `indexTaskDidFinish` calls does not have to relate to the number of `indexTasksWereScheduled` calls. - private let indexTaskDidFinish: @Sendable () -> Void + /// The number of `indexStatusDidChange` calls does not have to relate to the number of `indexTasksWereScheduled` calls. + private let indexStatusDidChange: @Sendable () -> Void // MARK: - Public API - /// The files that still need to be indexed. - /// - /// Scheduled tasks are files that are waiting for their target to be prepared or whose index store update task is - /// waiting to be scheduled by the task scheduler. - /// - /// `executing` are the files that currently have an active index store update task running. - public var inProgressIndexFiles: (scheduled: [DocumentURI], executing: [DocumentURI]) { - var scheduled: [DocumentURI] = [] - var executing: [DocumentURI] = [] - for (uri, status) in inProgressIndexTasks { - let isExecuting: Bool + /// A summary of the tasks that this `SemanticIndexManager` has currently scheduled or is currently indexing. + public var inProgressTasks: + ( + isGeneratingBuildGraph: Bool, + indexTasks: [DocumentURI: IndexTaskStatus], + preparationTasks: [ConfiguredTarget: IndexTaskStatus] + ) + { + let indexTasks = inProgressIndexTasks.mapValues { status in switch status { case .waitingForPreparation: - isExecuting = false + return IndexTaskStatus.scheduled case .updatingIndexStore(updateIndexStoreTask: let updateIndexStoreTask, indexTask: _): - isExecuting = updateIndexStoreTask.isExecuting - } - - if isExecuting { - executing.append(uri) - } else { - scheduled.append(uri) + return updateIndexStoreTask.isExecuting ? IndexTaskStatus.executing : IndexTaskStatus.scheduled } } - - return (scheduled, executing) + let preparationTasks = inProgressPreparationTasks.mapValues { queuedTask in + return queuedTask.isExecuting ? IndexTaskStatus.executing : IndexTaskStatus.scheduled + } + return (generateBuildGraphTask != nil, indexTasks, preparationTasks) } public init( @@ -151,14 +151,14 @@ public final actor SemanticIndexManager { testHooks: IndexTestHooks, indexTaskScheduler: TaskScheduler, indexTasksWereScheduled: @escaping @Sendable (Int) -> Void, - indexTaskDidFinish: @escaping @Sendable () -> Void + indexStatusDidChange: @escaping @Sendable () -> Void ) { self.index = index self.buildSystemManager = buildSystemManager self.testHooks = testHooks self.indexTaskScheduler = indexTaskScheduler self.indexTasksWereScheduled = indexTasksWereScheduled - self.indexTaskDidFinish = indexTaskDidFinish + self.indexStatusDidChange = indexStatusDidChange } /// Schedules a task to index `files`. Files that are known to be up-to-date based on `indexStatus` will @@ -358,6 +358,7 @@ public final actor SemanticIndexManager { } let preparationTask = await indexTaskScheduler.schedule(priority: priority, taskDescription) { task, newState in guard case .finished = newState else { + self.indexStatusDidChange() return } for target in targetsToPrepare { @@ -365,7 +366,7 @@ public final actor SemanticIndexManager { self.inProgressPreparationTasks[target] = nil } } - self.indexTaskDidFinish() + self.indexStatusDidChange() } for target in targetsToPrepare { inProgressPreparationTasks[target] = OpaqueQueuedIndexTask(preparationTask) @@ -400,6 +401,7 @@ public final actor SemanticIndexManager { ) let updateIndexTask = await indexTaskScheduler.schedule(priority: priority, taskDescription) { task, newState in guard case .finished = newState else { + self.indexStatusDidChange() return } for fileAndTarget in filesAndTargets { @@ -409,7 +411,7 @@ public final actor SemanticIndexManager { self.inProgressIndexTasks[fileAndTarget.file.sourceFile] = nil } } - self.indexTaskDidFinish() + self.indexStatusDidChange() } for fileAndTarget in filesAndTargets { if case .waitingForPreparation(preparationTaskID, let indexTask) = inProgressIndexTasks[ diff --git a/Sources/SourceKitLSP/IndexProgressManager.swift b/Sources/SourceKitLSP/IndexProgressManager.swift index fb758af73..0153f3ada 100644 --- a/Sources/SourceKitLSP/IndexProgressManager.swift +++ b/Sources/SourceKitLSP/IndexProgressManager.swift @@ -11,19 +11,13 @@ //===----------------------------------------------------------------------===// import LanguageServerProtocol +import SKCore import SKSupport import SemanticIndex /// Listens for index status updates from `SemanticIndexManagers`. From that information, it manages a /// `WorkDoneProgress` that communicates the index progress to the editor. actor IndexProgressManager { - /// A queue on which `indexTaskWasQueued` and `indexStatusDidChange` are handled. - /// - /// This allows the two functions two be `nonisolated` (and eg. the caller of `indexStatusDidChange` doesn't have to - /// wait for the work done progress to be updated) while still guaranteeing that we handle them in the order they - /// were called. - private let queue = AsyncQueue() - /// The `SourceKitLSPServer` for which this manages the index progress. It gathers all `SemanticIndexManagers` from /// the workspaces in the `SourceKitLSPServer`. private weak var sourceKitLSPServer: SourceKitLSPServer? @@ -47,20 +41,20 @@ actor IndexProgressManager { } /// Called when a new file is scheduled to be indexed. Increments the target index count, eg. the 3 in `1/3`. - nonisolated func indexTaskWasQueued(count: Int) { - queue.async { - await self.indexTaskWasQueuedImpl(count: count) + nonisolated func indexTasksWereScheduled(count: Int) { + Task { + await self.indexTasksWereScheduledImpl(count: count) } } - private func indexTaskWasQueuedImpl(count: Int) async { + private func indexTasksWereScheduledImpl(count: Int) async { queuedIndexTasks += count await indexStatusDidChangeImpl() } /// Called when a `SemanticIndexManager` finishes indexing a file. Adjusts the done index count, eg. the 1 in `1/3`. nonisolated func indexStatusDidChange() { - queue.async { + Task { await self.indexStatusDidChangeImpl() } } @@ -70,23 +64,47 @@ actor IndexProgressManager { workDoneProgress = nil return } - var scheduled: [DocumentURI] = [] - var executing: [DocumentURI] = [] + var isGeneratingBuildGraph = false + var indexTasks: [DocumentURI: IndexTaskStatus] = [:] + var preparationTasks: [ConfiguredTarget: IndexTaskStatus] = [:] for indexManager in await sourceKitLSPServer.workspaces.compactMap({ $0.semanticIndexManager }) { - let inProgress = await indexManager.inProgressIndexFiles - scheduled += inProgress.scheduled - executing += inProgress.executing + let inProgress = await indexManager.inProgressTasks + isGeneratingBuildGraph = isGeneratingBuildGraph || inProgress.isGeneratingBuildGraph + indexTasks.merge(inProgress.indexTasks) { lhs, rhs in + return max(lhs, rhs) + } + preparationTasks.merge(inProgress.preparationTasks) { lhs, rhs in + return max(lhs, rhs) + } } - if scheduled.isEmpty && executing.isEmpty { + if indexTasks.isEmpty { // Nothing left to index. Reset the target count and dismiss the work done progress. queuedIndexTasks = 0 workDoneProgress = nil return } - let finishedTasks = queuedIndexTasks - scheduled.count - executing.count - let message = "\(finishedTasks) / \(queuedIndexTasks)" + // We can get into a situation where queuedIndexTasks < indexTasks.count if we haven't processed all + // `indexTasksWereScheduled` calls yet but the semantic index managers already track them in their in-progress tasks. + // Clip the finished tasks to 0 because showing a negative number there looks stupid. + let finishedTasks = max(queuedIndexTasks - indexTasks.count, 0) + var message = "\(finishedTasks) / \(queuedIndexTasks)" + + if await sourceKitLSPServer.options.indexOptions.showActivePreparationTasksInProgress { + var inProgressTasks: [String] = [] + if isGeneratingBuildGraph { + inProgressTasks.append("- Generating build graph") + } + inProgressTasks += preparationTasks.filter { $0.value == .executing } + .map { "- Preparing \($0.key.targetID)" } + .sorted() + inProgressTasks += indexTasks.filter { $0.value == .executing } + .map { "- Indexing \($0.key.fileURL?.lastPathComponent ?? $0.key.pseudoPath)" } + .sorted() + + message += "\n\n" + inProgressTasks.joined(separator: "\n") + } let percentage = Int(Double(finishedTasks) / Double(queuedIndexTasks) * 100) if let workDoneProgress { diff --git a/Sources/SourceKitLSP/SourceKitLSPServer.swift b/Sources/SourceKitLSP/SourceKitLSPServer.swift index 210723007..95a84b160 100644 --- a/Sources/SourceKitLSP/SourceKitLSPServer.swift +++ b/Sources/SourceKitLSP/SourceKitLSPServer.swift @@ -1233,9 +1233,9 @@ extension SourceKitLSPServer { } }, indexTasksWereScheduled: { [weak self] count in - self?.indexProgressManager.indexTaskWasQueued(count: count) + self?.indexProgressManager.indexTasksWereScheduled(count: count) }, - indexTaskDidFinish: { [weak self] in + indexStatusDidChange: { [weak self] in self?.indexProgressManager.indexStatusDidChange() } ) @@ -1296,9 +1296,9 @@ extension SourceKitLSPServer { indexDelegate: nil, indexTaskScheduler: self.indexTaskScheduler, indexTasksWereScheduled: { [weak self] count in - self?.indexProgressManager.indexTaskWasQueued(count: count) + self?.indexProgressManager.indexTasksWereScheduled(count: count) }, - indexTaskDidFinish: { [weak self] in + indexStatusDidChange: { [weak self] in self?.indexProgressManager.indexStatusDidChange() } ) diff --git a/Sources/SourceKitLSP/Workspace.swift b/Sources/SourceKitLSP/Workspace.swift index 858ece66e..0deb0af2d 100644 --- a/Sources/SourceKitLSP/Workspace.swift +++ b/Sources/SourceKitLSP/Workspace.swift @@ -95,7 +95,7 @@ public final class Workspace: Sendable { indexDelegate: SourceKitIndexDelegate?, indexTaskScheduler: TaskScheduler, indexTasksWereScheduled: @escaping @Sendable (Int) -> Void, - indexTaskDidFinish: @escaping @Sendable () -> Void + indexStatusDidChange: @escaping @Sendable () -> Void ) async { self.documentManager = documentManager self.buildSetup = options.buildSetup @@ -115,7 +115,7 @@ public final class Workspace: Sendable { testHooks: options.indexTestHooks, indexTaskScheduler: indexTaskScheduler, indexTasksWereScheduled: indexTasksWereScheduled, - indexTaskDidFinish: indexTaskDidFinish + indexStatusDidChange: indexStatusDidChange ) } else { self.semanticIndexManager = nil @@ -153,7 +153,7 @@ public final class Workspace: Sendable { indexTaskScheduler: TaskScheduler, reloadPackageStatusCallback: @Sendable @escaping (ReloadPackageStatus) async -> Void, indexTasksWereScheduled: @Sendable @escaping (Int) -> Void, - indexTaskDidFinish: @Sendable @escaping () -> Void + indexStatusDidChange: @Sendable @escaping () -> Void ) async throws { var buildSystem: BuildSystem? = nil @@ -259,7 +259,7 @@ public final class Workspace: Sendable { indexDelegate: indexDelegate, indexTaskScheduler: indexTaskScheduler, indexTasksWereScheduled: indexTasksWereScheduled, - indexTaskDidFinish: indexTaskDidFinish + indexStatusDidChange: indexStatusDidChange ) } @@ -316,13 +316,22 @@ public struct IndexOptions: Sendable { /// Setting this to a value < 1 ensures that background indexing doesn't use all CPU resources. public var maxCoresPercentageToUseForBackgroundIndexing: Double + /// Whether to show the files that are currently being indexed / the targets that are currently being prepared in the + /// work done progress. + /// + /// This is an option because VS Code tries to render a multi-line work done progress into a single line text field in + /// the status bar, which looks broken. But at the same time, it is very useful to get a feeling about what's + /// currently happening indexing-wise. + public var showActivePreparationTasksInProgress: Bool + public init( indexStorePath: AbsolutePath? = nil, indexDatabasePath: AbsolutePath? = nil, indexPrefixMappings: [PathPrefixMapping]? = nil, listenToUnitEvents: Bool = true, enableBackgroundIndexing: Bool = false, - maxCoresPercentageToUseForBackgroundIndexing: Double = 1 + maxCoresPercentageToUseForBackgroundIndexing: Double = 1, + showActivePreparationTasksInProgress: Bool = false ) { self.indexStorePath = indexStorePath self.indexDatabasePath = indexDatabasePath @@ -330,5 +339,6 @@ public struct IndexOptions: Sendable { self.listenToUnitEvents = listenToUnitEvents self.enableBackgroundIndexing = enableBackgroundIndexing self.maxCoresPercentageToUseForBackgroundIndexing = maxCoresPercentageToUseForBackgroundIndexing + self.showActivePreparationTasksInProgress = showActivePreparationTasksInProgress } } diff --git a/Sources/sourcekit-lsp/SourceKitLSP.swift b/Sources/sourcekit-lsp/SourceKitLSP.swift index 89f768110..e309efbea 100644 --- a/Sources/sourcekit-lsp/SourceKitLSP.swift +++ b/Sources/sourcekit-lsp/SourceKitLSP.swift @@ -203,7 +203,15 @@ struct SourceKitLSP: AsyncParsableCommand { @Flag( help: "Enable background indexing. This feature is still under active development and may be incomplete." ) - var enableExperimentalBackgroundIndexing = false + var experimentalEnableBackgroundIndexing = false + + @Flag( + help: """ + Show which index tasks are currently running in the indexing work done progress. \ + This produces a multi-line work done progress, which might render incorrectly depending in the editor. + """ + ) + var experimentalShowActivePreparationTasksInProgress = false func mapOptions() -> SourceKitLSPServer.Options { var serverOptions = SourceKitLSPServer.Options() @@ -220,7 +228,8 @@ struct SourceKitLSP: AsyncParsableCommand { serverOptions.indexOptions.indexStorePath = indexStorePath serverOptions.indexOptions.indexDatabasePath = indexDatabasePath serverOptions.indexOptions.indexPrefixMappings = indexPrefixMappings - serverOptions.indexOptions.enableBackgroundIndexing = enableExperimentalBackgroundIndexing + serverOptions.indexOptions.enableBackgroundIndexing = experimentalEnableBackgroundIndexing + serverOptions.indexOptions.showActivePreparationTasksInProgress = experimentalShowActivePreparationTasksInProgress serverOptions.completionOptions.maxResults = completionMaxResults serverOptions.generatedInterfacesPath = generatedInterfacesPath diff --git a/Tests/SourceKitLSPTests/BackgroundIndexingTests.swift b/Tests/SourceKitLSPTests/BackgroundIndexingTests.swift index 6427e53e1..0b7bdaae4 100644 --- a/Tests/SourceKitLSPTests/BackgroundIndexingTests.swift +++ b/Tests/SourceKitLSPTests/BackgroundIndexingTests.swift @@ -462,7 +462,7 @@ final class BackgroundIndexingTests: XCTestCase { return } var didGetEndWorkDoneProgress = false - for _ in 0..<3 { + for _ in 0..<5 { let workEndProgress = try await project.testClient.nextNotification(ofType: WorkDoneProgress.self) switch workEndProgress.value { case .begin: diff --git a/Tests/SourceKitLSPTests/BuildSystemTests.swift b/Tests/SourceKitLSPTests/BuildSystemTests.swift index 49314ccac..8435c3c96 100644 --- a/Tests/SourceKitLSPTests/BuildSystemTests.swift +++ b/Tests/SourceKitLSPTests/BuildSystemTests.swift @@ -137,7 +137,7 @@ final class BuildSystemTests: XCTestCase { indexDelegate: nil, indexTaskScheduler: .forTesting, indexTasksWereScheduled: { _ in }, - indexTaskDidFinish: {} + indexStatusDidChange: {} ) await server.setWorkspaces([(workspace: workspace, isImplicit: false)]) From 3e6319c3b9f635fce915008eeb6e83a65cf7279c Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Tue, 21 May 2024 22:14:42 -0700 Subject: [PATCH 2/3] Produce an index log for the client This allows a user of SourceKit-LSP to inspect the result of background indexing. This allows a user of SourceKit-LSP to inspect the result of background indexing. I think this gives useful insights into what SourceKit-LSP is indexing and why/how it fails, if it fails, also for users of SourceKit-LSP. rdar://127474136 Fixes #1265 --- Sources/SKCore/BuildServerBuildSystem.swift | 5 +- Sources/SKCore/BuildSystem.swift | 5 +- Sources/SKCore/BuildSystemManager.swift | 7 +- Sources/SKCore/CMakeLists.txt | 1 + .../CompilationDatabaseBuildSystem.swift | 5 +- Sources/SKCore/IndexProcessResult.swift | 71 +++++++++++++++++++ .../SwiftPMBuildSystem.swift | 20 +++++- .../TestSourceKitLSPClient.swift | 21 ++---- .../PreparationTaskDescription.swift | 10 ++- .../SemanticIndex/SemanticIndexManager.swift | 10 +++ .../UpdateIndexStoreTaskDescription.swift | 19 ++++- Sources/SourceKitLSP/SourceKitLSPServer.swift | 23 +++++- Sources/SourceKitLSP/Workspace.swift | 4 ++ .../SKCoreTests/BuildSystemManagerTests.swift | 5 +- .../BackgroundIndexingTests.swift | 19 +++++ .../SourceKitLSPTests/BuildSystemTests.swift | 6 +- 16 files changed, 203 insertions(+), 28 deletions(-) create mode 100644 Sources/SKCore/IndexProcessResult.swift diff --git a/Sources/SKCore/BuildServerBuildSystem.swift b/Sources/SKCore/BuildServerBuildSystem.swift index 615ad3bd2..e2da82695 100644 --- a/Sources/SKCore/BuildServerBuildSystem.swift +++ b/Sources/SKCore/BuildServerBuildSystem.swift @@ -289,7 +289,10 @@ extension BuildServerBuildSystem: BuildSystem { return nil } - public func prepare(targets: [ConfiguredTarget]) async throws { + public func prepare( + targets: [ConfiguredTarget], + indexProcessDidProduceResult: @Sendable (IndexProcessResult) -> Void + ) async throws { throw PrepareNotSupportedError() } diff --git a/Sources/SKCore/BuildSystem.swift b/Sources/SKCore/BuildSystem.swift index e436cabc4..bc4848e82 100644 --- a/Sources/SKCore/BuildSystem.swift +++ b/Sources/SKCore/BuildSystem.swift @@ -158,7 +158,10 @@ public protocol BuildSystem: AnyObject, Sendable { /// Prepare the given targets for indexing and semantic functionality. This should build all swift modules of target /// dependencies. - func prepare(targets: [ConfiguredTarget]) async throws + func prepare( + targets: [ConfiguredTarget], + indexProcessDidProduceResult: @Sendable (IndexProcessResult) -> Void + ) async throws /// If the build system has knowledge about the language that this document should be compiled in, return it. /// diff --git a/Sources/SKCore/BuildSystemManager.swift b/Sources/SKCore/BuildSystemManager.swift index f2897bfa3..acba3e7f0 100644 --- a/Sources/SKCore/BuildSystemManager.swift +++ b/Sources/SKCore/BuildSystemManager.swift @@ -231,8 +231,11 @@ extension BuildSystemManager { return await buildSystem?.targets(dependingOn: targets) } - public func prepare(targets: [ConfiguredTarget]) async throws { - try await buildSystem?.prepare(targets: targets) + public func prepare( + targets: [ConfiguredTarget], + indexProcessDidProduceResult: @Sendable (IndexProcessResult) -> Void + ) async throws { + try await buildSystem?.prepare(targets: targets, indexProcessDidProduceResult: indexProcessDidProduceResult) } public func registerForChangeNotifications(for uri: DocumentURI, language: Language) async { diff --git a/Sources/SKCore/CMakeLists.txt b/Sources/SKCore/CMakeLists.txt index e207566f4..913e1e087 100644 --- a/Sources/SKCore/CMakeLists.txt +++ b/Sources/SKCore/CMakeLists.txt @@ -10,6 +10,7 @@ add_library(SKCore STATIC Debouncer.swift FallbackBuildSystem.swift FileBuildSettings.swift + IndexProcessResult.swift MainFilesProvider.swift PathPrefixMapping.swift SplitShellCommand.swift diff --git a/Sources/SKCore/CompilationDatabaseBuildSystem.swift b/Sources/SKCore/CompilationDatabaseBuildSystem.swift index 853765c54..ac5825f63 100644 --- a/Sources/SKCore/CompilationDatabaseBuildSystem.swift +++ b/Sources/SKCore/CompilationDatabaseBuildSystem.swift @@ -125,7 +125,10 @@ extension CompilationDatabaseBuildSystem: BuildSystem { return [ConfiguredTarget(targetID: "dummy", runDestinationID: "dummy")] } - public func prepare(targets: [ConfiguredTarget]) async throws { + public func prepare( + targets: [ConfiguredTarget], + indexProcessDidProduceResult: @Sendable (IndexProcessResult) -> Void + ) async throws { throw PrepareNotSupportedError() } diff --git a/Sources/SKCore/IndexProcessResult.swift b/Sources/SKCore/IndexProcessResult.swift new file mode 100644 index 000000000..07b71c7c8 --- /dev/null +++ b/Sources/SKCore/IndexProcessResult.swift @@ -0,0 +1,71 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import struct TSCBasic.ProcessResult + +/// Result of a process that prepares a target or updates the index store. To be shown in the build log. +/// +/// Abstracted over a `ProcessResult` to facilitate build systems that don't spawn a new process to prepare a target but +/// prepare it from a build graph they have loaded in-process. +public struct IndexProcessResult { + /// A human-readable description of what the process was trying to achieve, like `Preparing MyTarget` + public let taskDescription: String + + /// The command that was run to produce the result. + public let command: String + + /// The output that the process produced. + public let output: String + + /// Whether the process failed. + public let failed: Bool + + /// The duration it took for the process to execute. + public let duration: Duration + + public init(taskDescription: String, command: String, output: String, failed: Bool, duration: Duration) { + self.taskDescription = taskDescription + self.command = command + self.output = output + self.failed = failed + self.duration = duration + } + + public init(taskDescription: String, processResult: ProcessResult, start: ContinuousClock.Instant) { + let stdout = (try? String(bytes: processResult.output.get(), encoding: .utf8)) ?? "" + let stderr = (try? String(bytes: processResult.stderrOutput.get(), encoding: .utf8)) ?? "" + var outputComponents: [String] = [] + if !stdout.isEmpty { + outputComponents.append( + """ + Stdout: + \(stdout) + """ + ) + } + if !stderr.isEmpty { + outputComponents.append( + """ + Stderr: + \(stderr) + """ + ) + } + self.init( + taskDescription: taskDescription, + command: processResult.arguments.joined(separator: " "), + output: outputComponents.joined(separator: "\n\n"), + failed: processResult.exitStatus != .terminated(code: 0), + duration: start.duration(to: .now) + ) + } +} diff --git a/Sources/SKSwiftPMWorkspace/SwiftPMBuildSystem.swift b/Sources/SKSwiftPMWorkspace/SwiftPMBuildSystem.swift index c40cebe25..e54cbfe4d 100644 --- a/Sources/SKSwiftPMWorkspace/SwiftPMBuildSystem.swift +++ b/Sources/SKSwiftPMWorkspace/SwiftPMBuildSystem.swift @@ -459,17 +459,23 @@ extension SwiftPMBuildSystem: SKCore.BuildSystem { } } - public func prepare(targets: [ConfiguredTarget]) async throws { + public func prepare( + targets: [ConfiguredTarget], + indexProcessDidProduceResult: @Sendable (IndexProcessResult) -> Void + ) async throws { // TODO (indexing): Support preparation of multiple targets at once. // https://github.com/apple/sourcekit-lsp/issues/1262 for target in targets { - try await prepare(singleTarget: target) + try await prepare(singleTarget: target, indexProcessDidProduceResult: indexProcessDidProduceResult) } let filesInPreparedTargets = targets.flatMap { self.targets[$0]?.buildTarget.sources ?? [] } await fileDependenciesUpdatedDebouncer.scheduleCall(Set(filesInPreparedTargets.map(DocumentURI.init))) } - private func prepare(singleTarget target: ConfiguredTarget) async throws { + private func prepare( + singleTarget target: ConfiguredTarget, + indexProcessDidProduceResult: @Sendable (IndexProcessResult) -> Void + ) async throws { // TODO (indexing): Add a proper 'prepare' job in SwiftPM instead of building the target. // https://github.com/apple/sourcekit-lsp/issues/1254 guard let toolchain = await toolchainRegistry.default else { @@ -492,8 +498,16 @@ extension SwiftPMBuildSystem: SKCore.BuildSystem { if Task.isCancelled { return } + let start = ContinuousClock.now let process = try Process.launch(arguments: arguments, workingDirectory: nil) let result = try await process.waitUntilExitSendingSigIntOnTaskCancellation() + indexProcessDidProduceResult( + IndexProcessResult( + taskDescription: "Preparing \(target.targetID) for \(target.runDestinationID)", + processResult: result, + start: start + ) + ) switch result.exitStatus.exhaustivelySwitchable { case .terminated(code: 0): break diff --git a/Sources/SKTestSupport/TestSourceKitLSPClient.swift b/Sources/SKTestSupport/TestSourceKitLSPClient.swift index b3f348194..574ce1a45 100644 --- a/Sources/SKTestSupport/TestSourceKitLSPClient.swift +++ b/Sources/SKTestSupport/TestSourceKitLSPClient.swift @@ -243,25 +243,18 @@ public final class TestSourceKitLSPClient: MessageHandler { return try await nextNotification(ofType: PublishDiagnosticsNotification.self, timeout: timeout) } - private struct CastError: Error, CustomStringConvertible { - let expectedType: any NotificationType.Type - let actualType: any NotificationType.Type - - var description: String { "Expected a \(expectedType) but got '\(actualType)'" } - } - - /// Await the next diagnostic notification sent to the client. - /// - /// If the next notification is not of the expected type, this methods throws. + /// Waits for the next notification of the given type to be sent to the client. Ignores any notifications that are of + /// a different type. public func nextNotification( ofType: ExpectedNotificationType.Type, timeout: TimeInterval = defaultTimeout ) async throws -> ExpectedNotificationType { - let nextNotification = try await nextNotification(timeout: timeout) - guard let notification = nextNotification as? ExpectedNotificationType else { - throw CastError(expectedType: ExpectedNotificationType.self, actualType: type(of: nextNotification)) + while true { + let nextNotification = try await nextNotification(timeout: timeout) + if let notification = nextNotification as? ExpectedNotificationType { + return notification + } } - return notification } /// Handle the next request that is sent to the client with the given handler. diff --git a/Sources/SemanticIndex/PreparationTaskDescription.swift b/Sources/SemanticIndex/PreparationTaskDescription.swift index 8e7927e6b..bca2e35ba 100644 --- a/Sources/SemanticIndex/PreparationTaskDescription.swift +++ b/Sources/SemanticIndex/PreparationTaskDescription.swift @@ -37,6 +37,9 @@ public struct PreparationTaskDescription: IndexTaskDescription { private let preparationUpToDateStatus: IndexUpToDateStatusManager + /// See `SemanticIndexManager.indexProcessDidProduceResult` + private let indexProcessDidProduceResult: @Sendable (IndexProcessResult) -> Void + /// Test hooks that should be called when the preparation task finishes. private let testHooks: IndexTestHooks @@ -57,11 +60,13 @@ public struct PreparationTaskDescription: IndexTaskDescription { targetsToPrepare: [ConfiguredTarget], buildSystemManager: BuildSystemManager, preparationUpToDateStatus: IndexUpToDateStatusManager, + indexProcessDidProduceResult: @escaping @Sendable (IndexProcessResult) -> Void, testHooks: IndexTestHooks ) { self.targetsToPrepare = targetsToPrepare self.buildSystemManager = buildSystemManager self.preparationUpToDateStatus = preparationUpToDateStatus + self.indexProcessDidProduceResult = indexProcessDidProduceResult self.testHooks = testHooks } @@ -89,7 +94,10 @@ public struct PreparationTaskDescription: IndexTaskDescription { ) let startDate = Date() do { - try await buildSystemManager.prepare(targets: targetsToPrepare) + try await buildSystemManager.prepare( + targets: targetsToPrepare, + indexProcessDidProduceResult: indexProcessDidProduceResult + ) } catch { logger.error( "Preparation failed: \(error.forLogging)" diff --git a/Sources/SemanticIndex/SemanticIndexManager.swift b/Sources/SemanticIndex/SemanticIndexManager.swift index 7a50c88e7..631460253 100644 --- a/Sources/SemanticIndex/SemanticIndexManager.swift +++ b/Sources/SemanticIndex/SemanticIndexManager.swift @@ -108,6 +108,12 @@ public final actor SemanticIndexManager { /// workspaces. private let indexTaskScheduler: TaskScheduler + /// Callback to be called when the process to prepare a target finishes. + /// + /// Allows an index log to be displayed to the user that includes the command line invocations of all index-related + /// process launches, as well as their output. + private let indexProcessDidProduceResult: @Sendable (IndexProcessResult) -> Void + /// Called when files are scheduled to be indexed. /// /// The parameter is the number of files that were scheduled to be indexed. @@ -150,6 +156,7 @@ public final actor SemanticIndexManager { buildSystemManager: BuildSystemManager, testHooks: IndexTestHooks, indexTaskScheduler: TaskScheduler, + indexProcessDidProduceResult: @escaping @Sendable (IndexProcessResult) -> Void, indexTasksWereScheduled: @escaping @Sendable (Int) -> Void, indexStatusDidChange: @escaping @Sendable () -> Void ) { @@ -157,6 +164,7 @@ public final actor SemanticIndexManager { self.buildSystemManager = buildSystemManager self.testHooks = testHooks self.indexTaskScheduler = indexTaskScheduler + self.indexProcessDidProduceResult = indexProcessDidProduceResult self.indexTasksWereScheduled = indexTasksWereScheduled self.indexStatusDidChange = indexStatusDidChange } @@ -350,6 +358,7 @@ public final actor SemanticIndexManager { targetsToPrepare: targetsToPrepare, buildSystemManager: self.buildSystemManager, preparationUpToDateStatus: preparationUpToDateStatus, + indexProcessDidProduceResult: indexProcessDidProduceResult, testHooks: testHooks ) ) @@ -396,6 +405,7 @@ public final actor SemanticIndexManager { buildSystemManager: self.buildSystemManager, index: index, indexStoreUpToDateStatus: indexStoreUpToDateStatus, + indexProcessDidProduceResult: indexProcessDidProduceResult, testHooks: testHooks ) ) diff --git a/Sources/SemanticIndex/UpdateIndexStoreTaskDescription.swift b/Sources/SemanticIndex/UpdateIndexStoreTaskDescription.swift index 495fd9c2f..002252706 100644 --- a/Sources/SemanticIndex/UpdateIndexStoreTaskDescription.swift +++ b/Sources/SemanticIndex/UpdateIndexStoreTaskDescription.swift @@ -95,6 +95,9 @@ public struct UpdateIndexStoreTaskDescription: IndexTaskDescription { /// case we don't need to index it again. private let index: UncheckedIndex + /// See `SemanticIndexManager.indexProcessDidProduceResult` + private let indexProcessDidProduceResult: @Sendable (IndexProcessResult) -> Void + /// Test hooks that should be called when the index task finishes. private let testHooks: IndexTestHooks @@ -116,12 +119,14 @@ public struct UpdateIndexStoreTaskDescription: IndexTaskDescription { buildSystemManager: BuildSystemManager, index: UncheckedIndex, indexStoreUpToDateStatus: IndexUpToDateStatusManager, + indexProcessDidProduceResult: @escaping @Sendable (IndexProcessResult) -> Void, testHooks: IndexTestHooks ) { self.filesToIndex = filesToIndex self.buildSystemManager = buildSystemManager self.index = index self.indexStoreUpToDateStatus = indexStoreUpToDateStatus + self.indexProcessDidProduceResult = indexProcessDidProduceResult self.testHooks = testHooks } @@ -304,18 +309,28 @@ public struct UpdateIndexStoreTaskDescription: IndexTaskDescription { if Task.isCancelled { return } + let start = ContinuousClock.now let process = try Process.launch( arguments: processArguments, workingDirectory: workingDirectory ) let result = try await process.waitUntilExitSendingSigIntOnTaskCancellation() + + indexProcessDidProduceResult( + IndexProcessResult( + taskDescription: "Indexing \(indexFile.pseudoPath)", + processResult: result, + start: start + ) + ) + switch result.exitStatus.exhaustivelySwitchable { case .terminated(code: 0): break case .terminated(code: let code): // This most likely happens if there are compilation errors in the source file. This is nothing to worry about. - let stdout = (try? String(bytes: result.output.get(), encoding: .utf8)) ?? "" - let stderr = (try? String(bytes: result.stderrOutput.get(), encoding: .utf8)) ?? "" + let stdout = (try? String(bytes: result.output.get(), encoding: .utf8)) ?? "" + let stderr = (try? String(bytes: result.stderrOutput.get(), encoding: .utf8)) ?? "" // Indexing will frequently fail if the source code is in an invalid state. Thus, log the failure at a low level. logger.debug( """ diff --git a/Sources/SourceKitLSP/SourceKitLSPServer.swift b/Sources/SourceKitLSP/SourceKitLSPServer.swift index 95a84b160..644b3f093 100644 --- a/Sources/SourceKitLSP/SourceKitLSPServer.swift +++ b/Sources/SourceKitLSP/SourceKitLSPServer.swift @@ -698,7 +698,7 @@ public actor SourceKitLSPServer { } /// Send the given notification to the editor. - public func sendNotificationToClient(_ notification: some NotificationType) { + public nonisolated func sendNotificationToClient(_ notification: some NotificationType) { client.send(notification) } @@ -1177,6 +1177,21 @@ private extension LanguageServerProtocol.WorkspaceType { } } +extension SourceKitLSPServer { + nonisolated func indexTaskDidProduceResult(_ result: IndexProcessResult) { + self.sendNotificationToClient( + LogMessageNotification( + type: result.failed ? .info : .warning, + message: """ + \(result.taskDescription) finished in \(result.duration) + \(result.command) + \(result.output) + """ + ) + ) + } +} + // MARK: - Request and notification handling extension SourceKitLSPServer { @@ -1223,6 +1238,9 @@ extension SourceKitLSPServer { compilationDatabaseSearchPaths: self.options.compilationDatabaseSearchPaths, indexOptions: self.options.indexOptions, indexTaskScheduler: indexTaskScheduler, + indexProcessDidProduceResult: { [weak self] in + self?.indexTaskDidProduceResult($0) + }, reloadPackageStatusCallback: { [weak self] status in guard let self else { return } switch status { @@ -1295,6 +1313,9 @@ extension SourceKitLSPServer { index: nil, indexDelegate: nil, indexTaskScheduler: self.indexTaskScheduler, + indexProcessDidProduceResult: { [weak self] in + self?.indexTaskDidProduceResult($0) + }, indexTasksWereScheduled: { [weak self] count in self?.indexProgressManager.indexTasksWereScheduled(count: count) }, diff --git a/Sources/SourceKitLSP/Workspace.swift b/Sources/SourceKitLSP/Workspace.swift index 0deb0af2d..f913520e5 100644 --- a/Sources/SourceKitLSP/Workspace.swift +++ b/Sources/SourceKitLSP/Workspace.swift @@ -94,6 +94,7 @@ public final class Workspace: Sendable { index uncheckedIndex: UncheckedIndex?, indexDelegate: SourceKitIndexDelegate?, indexTaskScheduler: TaskScheduler, + indexProcessDidProduceResult: @escaping @Sendable (IndexProcessResult) -> Void, indexTasksWereScheduled: @escaping @Sendable (Int) -> Void, indexStatusDidChange: @escaping @Sendable () -> Void ) async { @@ -114,6 +115,7 @@ public final class Workspace: Sendable { buildSystemManager: buildSystemManager, testHooks: options.indexTestHooks, indexTaskScheduler: indexTaskScheduler, + indexProcessDidProduceResult: indexProcessDidProduceResult, indexTasksWereScheduled: indexTasksWereScheduled, indexStatusDidChange: indexStatusDidChange ) @@ -151,6 +153,7 @@ public final class Workspace: Sendable { compilationDatabaseSearchPaths: [RelativePath], indexOptions: IndexOptions = IndexOptions(), indexTaskScheduler: TaskScheduler, + indexProcessDidProduceResult: @escaping @Sendable (IndexProcessResult) -> Void, reloadPackageStatusCallback: @Sendable @escaping (ReloadPackageStatus) async -> Void, indexTasksWereScheduled: @Sendable @escaping (Int) -> Void, indexStatusDidChange: @Sendable @escaping () -> Void @@ -258,6 +261,7 @@ public final class Workspace: Sendable { index: UncheckedIndex(index), indexDelegate: indexDelegate, indexTaskScheduler: indexTaskScheduler, + indexProcessDidProduceResult: indexProcessDidProduceResult, indexTasksWereScheduled: indexTasksWereScheduled, indexStatusDidChange: indexStatusDidChange ) diff --git a/Tests/SKCoreTests/BuildSystemManagerTests.swift b/Tests/SKCoreTests/BuildSystemManagerTests.swift index 0caeef9ac..763991ecd 100644 --- a/Tests/SKCoreTests/BuildSystemManagerTests.swift +++ b/Tests/SKCoreTests/BuildSystemManagerTests.swift @@ -467,7 +467,10 @@ class ManualBuildSystem: BuildSystem { return [ConfiguredTarget(targetID: "dummy", runDestinationID: "dummy")] } - public func prepare(targets: [ConfiguredTarget]) async throws { + public func prepare( + targets: [ConfiguredTarget], + indexProcessDidProduceResult: @Sendable (IndexProcessResult) -> Void + ) async throws { throw PrepareNotSupportedError() } diff --git a/Tests/SourceKitLSPTests/BackgroundIndexingTests.swift b/Tests/SourceKitLSPTests/BackgroundIndexingTests.swift index 0b7bdaae4..240c2d86d 100644 --- a/Tests/SourceKitLSPTests/BackgroundIndexingTests.swift +++ b/Tests/SourceKitLSPTests/BackgroundIndexingTests.swift @@ -780,4 +780,23 @@ final class BackgroundIndexingTests: XCTestCase { allDocumentsOpened.fulfill() try await self.fulfillmentOfOrThrow([libDPreparedForEditing]) } + + public func testProduceIndexLog() async throws { + let project = try await SwiftPMTestProject( + files: [ + "MyFile.swift": "" + ], + serverOptions: backgroundIndexingOptions + ) + let targetPrepareNotification = try await project.testClient.nextNotification(ofType: LogMessageNotification.self) + XCTAssert( + targetPrepareNotification.message.hasPrefix("Preparing MyLibrary"), + "\(targetPrepareNotification.message) does not have the expected prefix" + ) + let indexFileNotification = try await project.testClient.nextNotification(ofType: LogMessageNotification.self) + XCTAssert( + indexFileNotification.message.hasPrefix("Indexing \(try project.uri(for: "MyFile.swift").pseudoPath)"), + "\(indexFileNotification.message) does not have the expected prefix" + ) + } } diff --git a/Tests/SourceKitLSPTests/BuildSystemTests.swift b/Tests/SourceKitLSPTests/BuildSystemTests.swift index 8435c3c96..378df97f8 100644 --- a/Tests/SourceKitLSPTests/BuildSystemTests.swift +++ b/Tests/SourceKitLSPTests/BuildSystemTests.swift @@ -59,7 +59,10 @@ actor TestBuildSystem: BuildSystem { return [ConfiguredTarget(targetID: "dummy", runDestinationID: "dummy")] } - public func prepare(targets: [ConfiguredTarget]) async throws { + public func prepare( + targets: [ConfiguredTarget], + indexProcessDidProduceResult: @Sendable (IndexProcessResult) -> Void + ) async throws { throw PrepareNotSupportedError() } @@ -136,6 +139,7 @@ final class BuildSystemTests: XCTestCase { index: nil, indexDelegate: nil, indexTaskScheduler: .forTesting, + indexProcessDidProduceResult: { _ in }, indexTasksWereScheduled: { _ in }, indexStatusDidChange: {} ) From da96d45443104c7c984590dbcca05a4045fb8f39 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Tue, 21 May 2024 22:27:36 -0700 Subject: [PATCH 3/3] Add a development subcommand to index a project This allows us to run `sourcekit-lsp index --project /path/to/project` to index a project. Intended to debugging purposes, eg. - Profile the time it takes to index a project - See if the project can be indexed successfully - Look at signposts generated during indexing in Instruments to see whether indexing or preparation is the bottleneck and how well we can parallelize tasks. --- Package.swift | 21 +++- Sources/CMakeLists.txt | 1 + Sources/Diagnose/CMakeLists.txt | 3 + Sources/Diagnose/IndexCommand.swift | 111 ++++++++++++++++++ Sources/InProcessClient/CMakeLists.txt | 14 +++ .../InProcessSourceKitLSPClient.swift | 81 +++++++++++++ .../LocalConnection.swift | 0 .../TestJSONRPCConnection.swift | 1 + .../TestSourceKitLSPClient.swift | 1 + Sources/sourcekit-lsp/SourceKitLSP.swift | 1 + 10 files changed, 233 insertions(+), 1 deletion(-) create mode 100644 Sources/Diagnose/IndexCommand.swift create mode 100644 Sources/InProcessClient/CMakeLists.txt create mode 100644 Sources/InProcessClient/InProcessSourceKitLSPClient.swift rename Sources/{LSPTestSupport => InProcessClient}/LocalConnection.swift (100%) diff --git a/Package.swift b/Package.swift index 7c0ddfb24..ebb59bd6a 100644 --- a/Package.swift +++ b/Package.swift @@ -74,9 +74,12 @@ let package = Package( .target( name: "Diagnose", dependencies: [ + "InProcessClient", "LSPLogging", - "SourceKitD", "SKCore", + "SKSupport", + "SourceKitD", + "SourceKitLSP", .product(name: "ArgumentParser", package: "swift-argument-parser"), .product(name: "SwiftIDEUtils", package: "swift-syntax"), .product(name: "SwiftSyntax", package: "swift-syntax"), @@ -99,6 +102,20 @@ let package = Package( ] ), + // MARK: InProcessClient + + .target( + name: "InProcessClient", + dependencies: [ + "CAtomics", + "LanguageServerProtocol", + "LSPLogging", + "SKCore", + "SourceKitLSP", + ], + exclude: ["CMakeLists.txt"] + ), + // MARK: LanguageServerProtocol // The core LSP types, suitable for any LSP implementation. .target( @@ -162,6 +179,7 @@ let package = Package( .target( name: "LSPTestSupport", dependencies: [ + "InProcessClient", "LanguageServerProtocol", "LanguageServerProtocolJSONRPC", "SKSupport", @@ -278,6 +296,7 @@ let package = Package( name: "SKTestSupport", dependencies: [ "CSKTestSupport", + "InProcessClient", "LanguageServerProtocol", "LSPTestSupport", "LSPLogging", diff --git a/Sources/CMakeLists.txt b/Sources/CMakeLists.txt index 4ed6c0350..2c729e5f0 100644 --- a/Sources/CMakeLists.txt +++ b/Sources/CMakeLists.txt @@ -2,6 +2,7 @@ add_subdirectory(BuildServerProtocol) add_subdirectory(CAtomics) add_subdirectory(Csourcekitd) add_subdirectory(Diagnose) +add_subdirectory(InProcessClient) add_subdirectory(LanguageServerProtocol) add_subdirectory(LanguageServerProtocolJSONRPC) add_subdirectory(LSPLogging) diff --git a/Sources/Diagnose/CMakeLists.txt b/Sources/Diagnose/CMakeLists.txt index e6c73f3ec..b93da89cd 100644 --- a/Sources/Diagnose/CMakeLists.txt +++ b/Sources/Diagnose/CMakeLists.txt @@ -2,6 +2,7 @@ add_library(Diagnose STATIC CommandConfiguration+Sendable.swift CommandLineArgumentsReducer.swift DiagnoseCommand.swift + IndexCommand.swift MergeSwiftFiles.swift OSLogScraper.swift ReduceCommand.swift @@ -23,6 +24,8 @@ set_target_properties(Diagnose PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${CMAKE_Swift_MODULE_DIRECTORY}) target_link_libraries(Diagnose PUBLIC + InProcessClient + LSPLogging SKCore SourceKitD ArgumentParser diff --git a/Sources/Diagnose/IndexCommand.swift b/Sources/Diagnose/IndexCommand.swift new file mode 100644 index 000000000..a00872d29 --- /dev/null +++ b/Sources/Diagnose/IndexCommand.swift @@ -0,0 +1,111 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import ArgumentParser +import Foundation +import InProcessClient +import LanguageServerProtocol +import SKCore +import SKSupport +import SourceKitLSP + +import struct TSCBasic.AbsolutePath +import class TSCBasic.Process +import var TSCBasic.stderrStream +import class TSCUtility.PercentProgressAnimation + +private actor IndexLogMessageHandler: MessageHandler { + var hasSeenError: Bool = false + + /// Queue to ensure that we don't have two interleaving `print` calls. + let queue = AsyncQueue() + + nonisolated func handle(_ notification: some NotificationType) { + if let notification = notification as? LogMessageNotification { + queue.async { + await self.handle(notification) + } + } + } + + func handle(_ notification: LogMessageNotification) { + self.hasSeenError = notification.type == .warning + print(notification.message) + } + + nonisolated func handle( + _ request: Request, + id: RequestID, + reply: @escaping @Sendable (LSPResult) -> Void + ) { + reply(.failure(.methodNotFound(Request.method))) + } + +} + +public struct IndexCommand: AsyncParsableCommand { + public static let configuration: CommandConfiguration = CommandConfiguration( + commandName: "index", + abstract: "Index a project and print all the processes executed for it as well as their outputs", + shouldDisplay: false + ) + + @Option( + name: .customLong("toolchain"), + help: """ + The toolchain used to reduce the sourcekitd issue. \ + If not specified, the toolchain is found in the same way that sourcekit-lsp finds it + """ + ) + var toolchainOverride: String? + + @Option(help: "The path to the project that should be indexed") + var project: String + + public init() {} + + public func run() async throws { + var serverOptions = SourceKitLSPServer.Options() + serverOptions.indexOptions.enableBackgroundIndexing = true + + let installPath = + if let toolchainOverride, let toolchain = Toolchain(try AbsolutePath(validating: toolchainOverride)) { + toolchain.path + } else { + try AbsolutePath(validating: Bundle.main.bundlePath) + } + + let messageHandler = IndexLogMessageHandler() + let inProcessClient = try await InProcessSourceKitLSPClient( + toolchainRegistry: ToolchainRegistry(installPath: installPath), + serverOptions: serverOptions, + workspaceFolders: [WorkspaceFolder(uri: DocumentURI(URL(fileURLWithPath: project)))], + messageHandler: messageHandler + ) + let start = ContinuousClock.now + _ = try await inProcessClient.send(PollIndexRequest()) + print("Indexing finished in \(start.duration(to: .now))") + if await messageHandler.hasSeenError { + throw ExitCode(1) + } + } +} + +fileprivate extension SourceKitLSPServer { + func handle(_ request: R, requestID: RequestID) async throws -> R.Response { + return try await withCheckedThrowingContinuation { continuation in + self.handle(request, id: requestID) { result in + continuation.resume(with: result) + } + } + } +} diff --git a/Sources/InProcessClient/CMakeLists.txt b/Sources/InProcessClient/CMakeLists.txt new file mode 100644 index 000000000..023f44d31 --- /dev/null +++ b/Sources/InProcessClient/CMakeLists.txt @@ -0,0 +1,14 @@ +add_library(InProcessClient STATIC + InProcessSourceKitLSPClient.swift + LocalConnection.swift) + +set_target_properties(InProcessClient PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES ${CMAKE_Swift_MODULE_DIRECTORY}) + +target_link_libraries(InProcessClient PUBLIC + CAtomics + LanguageServerProtocol + LSPLogging + SKCore + SourceKitLSP +) diff --git a/Sources/InProcessClient/InProcessSourceKitLSPClient.swift b/Sources/InProcessClient/InProcessSourceKitLSPClient.swift new file mode 100644 index 000000000..173e99b18 --- /dev/null +++ b/Sources/InProcessClient/InProcessSourceKitLSPClient.swift @@ -0,0 +1,81 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import CAtomics +import LanguageServerProtocol +import SKCore +import SourceKitLSP + +/// Launches a `SourceKitLSPServer` in-process and allows sending messages to it. +public final class InProcessSourceKitLSPClient: Sendable { + private let server: SourceKitLSPServer + + /// `nonisolated(unsafe)` if fine because `nextRequestID` is atomic. + private nonisolated(unsafe) var nextRequestID = AtomicUInt32(initialValue: 0) + + /// Create a new `SourceKitLSPServer`. An `InitializeRequest` is automatically sent to the server. + /// + /// `messageHandler` handles notifications and requests sent from the SourceKit-LSP server to the client. + public init( + toolchainRegistry: ToolchainRegistry, + serverOptions: SourceKitLSPServer.Options = SourceKitLSPServer.Options(), + capabilities: ClientCapabilities = ClientCapabilities(), + workspaceFolders: [WorkspaceFolder], + messageHandler: any MessageHandler + ) async throws { + let serverToClientConnection = LocalConnection(name: "client") + self.server = SourceKitLSPServer( + client: serverToClientConnection, + toolchainRegistry: toolchainRegistry, + options: serverOptions, + onExit: { + serverToClientConnection.close() + } + ) + serverToClientConnection.start(handler: messageHandler) + _ = try await self.send( + InitializeRequest( + processId: nil, + rootPath: nil, + rootURI: nil, + initializationOptions: nil, + capabilities: capabilities, + trace: .off, + workspaceFolders: workspaceFolders + ) + ) + } + + /// Send the request to `server` and return the request result. + /// + /// - Important: Because this is an async function, Swift concurrency makes no guarantees about the execution ordering + /// of this request with regard to other requests to the server. If execution of requests in a particular order is + /// necessary and the response of the request is not awaited, use the version of the function that takes a + /// completion handler + public func send(_ request: R) async throws -> R.Response { + return try await withCheckedThrowingContinuation { continuation in + self.send(request) { + continuation.resume(with: $0) + } + } + } + + /// Send the request to `server` and return the request result via a completion handler. + public func send(_ request: R, reply: @Sendable @escaping (LSPResult) -> Void) { + server.handle(request, id: .number(Int(nextRequestID.fetchAndIncrement())), reply: reply) + } + + /// Send the notification to `server`. + public func send(_ notification: some NotificationType) { + server.handle(notification) + } +} diff --git a/Sources/LSPTestSupport/LocalConnection.swift b/Sources/InProcessClient/LocalConnection.swift similarity index 100% rename from Sources/LSPTestSupport/LocalConnection.swift rename to Sources/InProcessClient/LocalConnection.swift diff --git a/Sources/LSPTestSupport/TestJSONRPCConnection.swift b/Sources/LSPTestSupport/TestJSONRPCConnection.swift index 9400822cf..992ec51fe 100644 --- a/Sources/LSPTestSupport/TestJSONRPCConnection.swift +++ b/Sources/LSPTestSupport/TestJSONRPCConnection.swift @@ -10,6 +10,7 @@ // //===----------------------------------------------------------------------===// +import InProcessClient import LanguageServerProtocol import LanguageServerProtocolJSONRPC import SKSupport diff --git a/Sources/SKTestSupport/TestSourceKitLSPClient.swift b/Sources/SKTestSupport/TestSourceKitLSPClient.swift index 574ce1a45..04009d2a1 100644 --- a/Sources/SKTestSupport/TestSourceKitLSPClient.swift +++ b/Sources/SKTestSupport/TestSourceKitLSPClient.swift @@ -12,6 +12,7 @@ import CAtomics import Foundation +import InProcessClient import LSPTestSupport import LanguageServerProtocol import LanguageServerProtocolJSONRPC diff --git a/Sources/sourcekit-lsp/SourceKitLSP.swift b/Sources/sourcekit-lsp/SourceKitLSP.swift index e309efbea..57c63eec8 100644 --- a/Sources/sourcekit-lsp/SourceKitLSP.swift +++ b/Sources/sourcekit-lsp/SourceKitLSP.swift @@ -105,6 +105,7 @@ struct SourceKitLSP: AsyncParsableCommand { abstract: "Language Server Protocol implementation for Swift and C-based languages", subcommands: [ DiagnoseCommand.self, + IndexCommand.self, ReduceCommand.self, ReduceFrontendCommand.self, SourceKitdRequestCommand.self,