From 2c616f7838013d36a7899163fe011cdd432a9669 Mon Sep 17 00:00:00 2001 From: Marcin Krzyzanowski Date: Tue, 26 Dec 2023 22:55:12 +0100 Subject: [PATCH 1/3] Refactor SemanticTokenTypes, SemanticTokenModifiers --- .../SupportTypes/SemanticTokenModifiers.swift | 67 +++++++ .../SupportTypes/SemanticTokenTypes.swift | 77 ++++++++ .../Array+SyntaxHighlightingToken.swift | 2 +- .../Swift/SwiftLanguageServer.swift | 4 +- .../Swift/SyntaxHighlightingToken.swift | 165 +++--------------- .../SemanticTokensTests.swift | 8 +- 6 files changed, 175 insertions(+), 148 deletions(-) create mode 100644 Sources/LanguageServerProtocol/SupportTypes/SemanticTokenModifiers.swift create mode 100644 Sources/LanguageServerProtocol/SupportTypes/SemanticTokenTypes.swift diff --git a/Sources/LanguageServerProtocol/SupportTypes/SemanticTokenModifiers.swift b/Sources/LanguageServerProtocol/SupportTypes/SemanticTokenModifiers.swift new file mode 100644 index 000000000..a64de0515 --- /dev/null +++ b/Sources/LanguageServerProtocol/SupportTypes/SemanticTokenModifiers.swift @@ -0,0 +1,67 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Foundation + +/// Additional metadata about a token. +/// +/// Similar to `SemanticTokenTypes`, the bit indices should +/// be numbered starting at 0. +public struct SemanticTokenModifiers: OptionSet, Hashable { + public let rawValue: UInt32 + + public init(rawValue: UInt32) { + self.rawValue = rawValue + } + + public static let declaration = Self(rawValue: 1 << 0) + public static let definition = Self(rawValue: 1 << 1) + public static let readonly = Self(rawValue: 1 << 2) + public static let `static` = Self(rawValue: 1 << 3) + public static let deprecated = Self(rawValue: 1 << 4) + public static let abstract = Self(rawValue: 1 << 5) + public static let async = Self(rawValue: 1 << 6) + public static let modification = Self(rawValue: 1 << 7) + public static let documentation = Self(rawValue: 1 << 8) + public static let defaultLibrary = Self(rawValue: 1 << 9) + + public var name: String? { + switch self { + case .declaration: return "declaration" + case .definition: return "definition" + case .readonly: return "readonly" + case .static: return "static" + case .deprecated: return "deprecated" + case .abstract: return "abstract" + case .async: return "async" + case .modification: return "modification" + case .documentation: return "documentation" + case .defaultLibrary: return "defaultLibrary" + default: return nil + } + } + + /// All available modifiers, in ascending order of the bit index + /// they are represented with (starting at the rightmost bit). + public static let predefined: [Self] = [ + .declaration, + .definition, + .readonly, + .static, + .deprecated, + .abstract, + .async, + .modification, + .documentation, + .defaultLibrary, + ] +} diff --git a/Sources/LanguageServerProtocol/SupportTypes/SemanticTokenTypes.swift b/Sources/LanguageServerProtocol/SupportTypes/SemanticTokenTypes.swift new file mode 100644 index 000000000..f03c06644 --- /dev/null +++ b/Sources/LanguageServerProtocol/SupportTypes/SemanticTokenTypes.swift @@ -0,0 +1,77 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Foundation + +/// The predefined token type values +/// +/// The protocol defines a set of token types and modifiers but clients are +/// allowed to extend these and announce the values they support in the +/// corresponding client capability. +public struct SemanticTokenTypes: Hashable { + public let name: String + public init(_ name: String) { + self.name = name + } + + public static let namespace = Self("namespace") + /// Represents a generic type. Acts as a fallback for types which + /// can't be mapped to a specific type like class or enum. + public static let type = Self("type") + public static let `class` = Self("class") + public static let `enum` = Self("enum") + public static let interface = Self("interface") + public static let `struct` = Self("struct") + public static let typeParameter = Self("typeParameter") + public static let parameter = Self("parameter") + public static let variable = Self("variable") + public static let property = Self("property") + public static let enumMember = Self("enumMember") + public static let event = Self("event") + public static let function = Self("function") + public static let method = Self("method") + public static let macro = Self("macro") + public static let keyword = Self("keyword") + public static let modifier = Self("modifier") + public static let comment = Self("comment") + public static let string = Self("string") + public static let number = Self("number") + public static let regexp = Self("regexp") + public static let `operator` = Self("operator") + /// since 3.17.0 + public static let decorator = Self("decorator") + + public static var predefined: [Self] = [ + .namespace, + .type, + .class, + .enum, + .interface, + .struct, + .typeParameter, + .parameter, + .variable, + .property, + .enumMember, + .event, + .function, + .method, + .macro, + .keyword, + .modifier, + .comment, + .string, + .number, + .regexp, + .operator, + ] +} diff --git a/Sources/SKTestSupport/Array+SyntaxHighlightingToken.swift b/Sources/SKTestSupport/Array+SyntaxHighlightingToken.swift index 92f7692fc..d92d8363e 100644 --- a/Sources/SKTestSupport/Array+SyntaxHighlightingToken.swift +++ b/Sources/SKTestSupport/Array+SyntaxHighlightingToken.swift @@ -37,7 +37,7 @@ extension Array where Element == SyntaxHighlightingToken { current.utf16index = charDelta } - guard let kind = SyntaxHighlightingToken.Kind(rawValue: rawKind) else { continue } + let kind = SyntaxHighlightingToken.Kind.all[Int(rawKind)] let modifiers = SyntaxHighlightingToken.Modifiers(rawValue: rawModifiers) append( diff --git a/Sources/SourceKitLSP/Swift/SwiftLanguageServer.swift b/Sources/SourceKitLSP/Swift/SwiftLanguageServer.swift index 522810f37..685f119fc 100644 --- a/Sources/SourceKitLSP/Swift/SwiftLanguageServer.swift +++ b/Sources/SourceKitLSP/Swift/SwiftLanguageServer.swift @@ -231,8 +231,8 @@ extension SwiftLanguageServer { ), semanticTokensProvider: SemanticTokensOptions( legend: SemanticTokensLegend( - tokenTypes: SyntaxHighlightingToken.Kind.allCases.map(\.lspName), - tokenModifiers: SyntaxHighlightingToken.Modifiers.allModifiers.map { $0.lspName! } + tokenTypes: SyntaxHighlightingToken.Kind.all.map(\.name), + tokenModifiers: SyntaxHighlightingToken.Modifiers.all.compactMap(\.name) ), range: .bool(true), full: .bool(true) diff --git a/Sources/SourceKitLSP/Swift/SyntaxHighlightingToken.swift b/Sources/SourceKitLSP/Swift/SyntaxHighlightingToken.swift index 50b3186b0..a6a8c09fc 100644 --- a/Sources/SourceKitLSP/Swift/SyntaxHighlightingToken.swift +++ b/Sources/SourceKitLSP/Swift/SyntaxHighlightingToken.swift @@ -47,146 +47,8 @@ public struct SyntaxHighlightingToken: Hashable { self.init(range: range, kind: kind, modifiers: modifiers) } - /// The token type. - /// - /// Represented using an int to make the conversion to - /// LSP tokens efficient. The order of this enum does not have to be - /// stable, since we provide a `SemanticTokensLegend` during initialization. - /// It is, however, important that the values are numbered from 0 due to - /// the way the kinds are encoded in LSP. - /// Also note that we intentionally use an enum here instead of e.g. a - /// `RawRepresentable` struct, since we want to have a conversion to - /// strings for known kinds and since these kinds are only provided by the - /// server, i.e. there is no need to handle cases where unknown kinds - /// have to be decoded. - public enum Kind: UInt32, CaseIterable, Hashable { - case namespace = 0 - case type - case actor - case `class` - case `enum` - case interface - case `struct` - case typeParameter - case parameter - case variable - case property - case enumMember - case event - case function - case method - case macro - case keyword - case modifier - case comment - case string - case number - case regexp - case `operator` - case decorator - /// **(LSP Extension)** - case identifier - - /// The name of the token type used by LSP. - var lspName: String { - switch self { - case .namespace: return "namespace" - case .type: return "type" - case .actor: return "class" // LSP doesn’t know about actors. Display actors as classes. - case .class: return "class" - case .enum: return "enum" - case .interface: return "interface" - case .struct: return "struct" - case .typeParameter: return "typeParameter" - case .parameter: return "parameter" - case .variable: return "variable" - case .property: return "property" - case .enumMember: return "enumMember" - case .event: return "event" - case .function: return "function" - case .method: return "method" - case .macro: return "macro" - case .keyword: return "keyword" - case .modifier: return "modifier" - case .comment: return "comment" - case .string: return "string" - case .number: return "number" - case .regexp: return "regexp" - case .operator: return "operator" - case .decorator: return "decorator" - case .identifier: return "identifier" - } - } - - /// **Public for testing** - public var _lspName: String { - lspName - } - } - - /// Additional metadata about a token. - /// - /// Similar to `Kind`, the raw values do not actually have - /// to be stable, do note however that the bit indices should - /// be numbered starting at 0 and that the ordering should - /// correspond to `allModifiers`. - public struct Modifiers: OptionSet, Hashable { - public static let declaration = Self(rawValue: 1 << 0) - public static let definition = Self(rawValue: 1 << 1) - public static let readonly = Self(rawValue: 1 << 2) - public static let `static` = Self(rawValue: 1 << 3) - public static let deprecated = Self(rawValue: 1 << 4) - public static let abstract = Self(rawValue: 1 << 5) - public static let async = Self(rawValue: 1 << 6) - public static let modification = Self(rawValue: 1 << 7) - public static let documentation = Self(rawValue: 1 << 8) - public static let defaultLibrary = Self(rawValue: 1 << 9) - - /// All available modifiers, in ascending order of the bit index - /// they are represented with (starting at the rightmost bit). - public static let allModifiers: [Self] = [ - .declaration, - .definition, - .readonly, - .static, - .deprecated, - .abstract, - .async, - .modification, - .documentation, - .defaultLibrary, - ] - - public let rawValue: UInt32 - - /// The name of the modifier used by LSP, if this - /// is a single modifier. Note that every modifier - /// in `allModifiers` must have an associated `lspName`. - var lspName: String? { - switch self { - case .declaration: return "declaration" - case .definition: return "definition" - case .readonly: return "readonly" - case .static: return "static" - case .deprecated: return "deprecated" - case .abstract: return "abstract" - case .async: return "async" - case .modification: return "modification" - case .documentation: return "documentation" - case .defaultLibrary: return "defaultLibrary" - default: return nil - } - } - - /// **Public for testing** - public var _lspName: String? { - lspName - } - - public init(rawValue: UInt32) { - self.rawValue = rawValue - } - } + public typealias Kind = SemanticTokenTypes + public typealias Modifiers = SemanticTokenModifiers } extension Array where Element == SyntaxHighlightingToken { @@ -214,7 +76,7 @@ extension Array where Element == SyntaxHighlightingToken { UInt32(lineDelta), UInt32(charDelta), UInt32(token.utf16length), - token.kind.rawValue, + token.kind.tokenType, token.modifiers.rawValue, ] } @@ -230,3 +92,24 @@ extension Array where Element == SyntaxHighlightingToken { return filter { !otherRanges.contains($0.range) } + other } } + +extension SemanticTokenTypes { + /// **(LSP Extension)** + public static let identifier = Self("identifier") + + // LSP doesn’t know about actors. Display actors as classes. + public static let actor = Self("class") + + /// All tokens supported by sourcekit-lsp + public static let all: [Self] = predefined + [.identifier, .actor] + + /// Token types are looked up by index + public var tokenType: UInt32 { + UInt32(Self.all.firstIndex(of: self)!) + } +} + +extension SemanticTokenModifiers { + /// All tokens supported by sourcekit-lsp + public static let all: [Self] = predefined +} diff --git a/Tests/SourceKitLSPTests/SemanticTokensTests.swift b/Tests/SourceKitLSPTests/SemanticTokensTests.swift index 2ee4a07b9..424a2f21b 100644 --- a/Tests/SourceKitLSPTests/SemanticTokensTests.swift +++ b/Tests/SourceKitLSPTests/SemanticTokensTests.swift @@ -53,8 +53,8 @@ final class SemanticTokensTests: XCTestCase { range: .bool(true), full: .bool(true) ), - tokenTypes: Token.Kind.allCases.map(\._lspName), - tokenModifiers: Token.Modifiers.allModifiers.map { $0._lspName! }, + tokenTypes: Token.Kind.all.map(\.name), + tokenModifiers: Token.Modifiers.all.compactMap(\.name), formats: [.relative] ) ) @@ -176,13 +176,13 @@ final class SemanticTokensTests: XCTestCase { 2, // line delta 3, // char delta 5, // length - Token.Kind.string.rawValue, // kind + Token.Kind.string.tokenType, // kind 0, // modifiers 2, // line delta 2, // char delta 1, // length - Token.Kind.interface.rawValue, // kind + Token.Kind.interface.tokenType, // kind Token.Modifiers.deprecated.rawValue | Token.Modifiers.definition.rawValue, // modifiers ] ) From 0797a361d804b08959970ac569fb55ec1bdf7cd8 Mon Sep 17 00:00:00 2001 From: Marcin Krzyzanowski Date: Wed, 27 Dec 2023 20:23:44 +0100 Subject: [PATCH 2/3] Add files to CMake files --- Sources/LanguageServerProtocol/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Sources/LanguageServerProtocol/CMakeLists.txt b/Sources/LanguageServerProtocol/CMakeLists.txt index 47fb86fde..24ee3b108 100644 --- a/Sources/LanguageServerProtocol/CMakeLists.txt +++ b/Sources/LanguageServerProtocol/CMakeLists.txt @@ -116,7 +116,9 @@ add_library(LanguageServerProtocol STATIC SupportTypes/PositionEncoding.swift SupportTypes/ProgressToken.swift SupportTypes/RegistrationOptions.swift + SupportTypes/SemanticTokenModifiers.swift SupportTypes/SemanticTokens.swift + SupportTypes/SemanticTokenTypes.swift SupportTypes/ServerCapabilities.swift SupportTypes/SKCompletionOptions.swift SupportTypes/StringOrMarkupContent.swift From 79ca4c0ac628cb4d8618e016a359978455e48c2a Mon Sep 17 00:00:00 2001 From: Marcin Krzyzanowski Date: Mon, 8 Jan 2024 23:51:09 +0100 Subject: [PATCH 3/3] Remove redundant typealias --- .../Array+SyntaxHighlightingToken.swift | 4 ++-- Sources/SourceKitLSP/Swift/SemanticTokens.swift | 2 +- .../SourceKitLSP/Swift/SwiftLanguageServer.swift | 4 ++-- .../Swift/SyntaxHighlightingToken.swift | 11 ++++------- .../Swift/SyntaxHighlightingTokenParser.swift | 2 +- Tests/SourceKitLSPTests/SemanticTokensTests.swift | 14 +++++++------- 6 files changed, 17 insertions(+), 20 deletions(-) diff --git a/Sources/SKTestSupport/Array+SyntaxHighlightingToken.swift b/Sources/SKTestSupport/Array+SyntaxHighlightingToken.swift index d92d8363e..bc12294bc 100644 --- a/Sources/SKTestSupport/Array+SyntaxHighlightingToken.swift +++ b/Sources/SKTestSupport/Array+SyntaxHighlightingToken.swift @@ -37,8 +37,8 @@ extension Array where Element == SyntaxHighlightingToken { current.utf16index = charDelta } - let kind = SyntaxHighlightingToken.Kind.all[Int(rawKind)] - let modifiers = SyntaxHighlightingToken.Modifiers(rawValue: rawModifiers) + let kind = SemanticTokenTypes.all[Int(rawKind)] + let modifiers = SemanticTokenModifiers(rawValue: rawModifiers) append( SyntaxHighlightingToken( diff --git a/Sources/SourceKitLSP/Swift/SemanticTokens.swift b/Sources/SourceKitLSP/Swift/SemanticTokens.swift index b4f26914e..fa0de03ac 100644 --- a/Sources/SourceKitLSP/Swift/SemanticTokens.swift +++ b/Sources/SourceKitLSP/Swift/SemanticTokens.swift @@ -129,7 +129,7 @@ extension SyntaxClassifiedRange { } extension SyntaxClassification { - fileprivate var highlightingKindAndModifiers: (SyntaxHighlightingToken.Kind, SyntaxHighlightingToken.Modifiers)? { + fileprivate var highlightingKindAndModifiers: (SemanticTokenTypes, SemanticTokenModifiers)? { switch self { case .none: return nil diff --git a/Sources/SourceKitLSP/Swift/SwiftLanguageServer.swift b/Sources/SourceKitLSP/Swift/SwiftLanguageServer.swift index 685f119fc..94a1e3e70 100644 --- a/Sources/SourceKitLSP/Swift/SwiftLanguageServer.swift +++ b/Sources/SourceKitLSP/Swift/SwiftLanguageServer.swift @@ -231,8 +231,8 @@ extension SwiftLanguageServer { ), semanticTokensProvider: SemanticTokensOptions( legend: SemanticTokensLegend( - tokenTypes: SyntaxHighlightingToken.Kind.all.map(\.name), - tokenModifiers: SyntaxHighlightingToken.Modifiers.all.compactMap(\.name) + tokenTypes: SemanticTokenTypes.all.map(\.name), + tokenModifiers: SemanticTokenModifiers.all.compactMap(\.name) ), range: .bool(true), full: .bool(true) diff --git a/Sources/SourceKitLSP/Swift/SyntaxHighlightingToken.swift b/Sources/SourceKitLSP/Swift/SyntaxHighlightingToken.swift index a6a8c09fc..40afe3590 100644 --- a/Sources/SourceKitLSP/Swift/SyntaxHighlightingToken.swift +++ b/Sources/SourceKitLSP/Swift/SyntaxHighlightingToken.swift @@ -23,9 +23,9 @@ public struct SyntaxHighlightingToken: Hashable { } } /// The token type. - public var kind: Kind + public var kind: SemanticTokenTypes /// Additional metadata about the token. - public var modifiers: Modifiers + public var modifiers: SemanticTokenModifiers /// The (inclusive) start position of the token. public var start: Position { range.lowerBound } @@ -34,7 +34,7 @@ public struct SyntaxHighlightingToken: Hashable { /// The length of the token in UTF-16 code units. public var utf16length: Int { end.utf16index - start.utf16index } - public init(range: Range, kind: Kind, modifiers: Modifiers = []) { + public init(range: Range, kind: SemanticTokenTypes, modifiers: SemanticTokenModifiers = []) { assert(range.lowerBound.line == range.upperBound.line) self.range = range @@ -42,13 +42,10 @@ public struct SyntaxHighlightingToken: Hashable { self.modifiers = modifiers } - public init(start: Position, utf16length: Int, kind: Kind, modifiers: Modifiers = []) { + public init(start: Position, utf16length: Int, kind: SemanticTokenTypes, modifiers: SemanticTokenModifiers = []) { let range = start.. (SyntaxHighlightingToken.Kind, SyntaxHighlightingToken.Modifiers)? { + ) -> (SemanticTokenTypes, SemanticTokenModifiers)? { let api = sourcekitd.api let values = sourcekitd.values switch uid { diff --git a/Tests/SourceKitLSPTests/SemanticTokensTests.swift b/Tests/SourceKitLSPTests/SemanticTokensTests.swift index 424a2f21b..ac341fc60 100644 --- a/Tests/SourceKitLSPTests/SemanticTokensTests.swift +++ b/Tests/SourceKitLSPTests/SemanticTokensTests.swift @@ -53,8 +53,8 @@ final class SemanticTokensTests: XCTestCase { range: .bool(true), full: .bool(true) ), - tokenTypes: Token.Kind.all.map(\.name), - tokenModifiers: Token.Modifiers.all.compactMap(\.name), + tokenTypes: SemanticTokenTypes.all.map(\.name), + tokenModifiers: SemanticTokenModifiers.all.compactMap(\.name), formats: [.relative] ) ) @@ -176,14 +176,14 @@ final class SemanticTokensTests: XCTestCase { 2, // line delta 3, // char delta 5, // length - Token.Kind.string.tokenType, // kind + SemanticTokenTypes.string.tokenType, // kind 0, // modifiers 2, // line delta 2, // char delta 1, // length - Token.Kind.interface.tokenType, // kind - Token.Modifiers.deprecated.rawValue | Token.Modifiers.definition.rawValue, // modifiers + SemanticTokenTypes.interface.tokenType, // kind + SemanticTokenModifiers.deprecated.rawValue | SemanticTokenModifiers.definition.rawValue, // modifiers ] ) @@ -886,8 +886,8 @@ extension Token { line: Int, utf16index: Int, length: Int, - kind: Token.Kind, - modifiers: Token.Modifiers = [] + kind: SemanticTokenTypes, + modifiers: SemanticTokenModifiers = [] ) { self.init( start: Position(line: line, utf16index: utf16index),