From 16e03ec25fdff92f7b704fd08d50593566b1c5f2 Mon Sep 17 00:00:00 2001 From: Khan Winter <35942988+thecoolwinter@users.noreply.github.com> Date: Fri, 23 Sep 2022 10:41:36 -0500 Subject: [PATCH 1/4] Draft Ready --- .../CodeEditTextView/Enums/CaptureName.swift | 47 ++++ .../NSRange+/NSRange+InputEdit.swift | 32 +++ .../NSRange+/NSRange+NSTextRange.swift | 24 ++ .../NSRange+String.swift} | 0 .../STTextView+/STTextView+VisibleRange.swift | 31 +++ .../Highlighting/HighlightRange.swift | 19 ++ .../Highlighting/Highlighter.swift | 245 ++++++++++++++++++ .../STTextViewController+CaptureNames.swift | 43 --- ...extViewController+STTextViewDelegate.swift | 39 --- .../STTextViewController+TreeSitter.swift | 96 ------- .../STTextViewController.swift | 58 ++++- .../CodeEditTextView/Theme/EditorTheme.swift | 41 +++ .../TreeSitter/TreeSitterClient.swift | 175 +++++++++++++ 13 files changed, 661 insertions(+), 189 deletions(-) create mode 100644 Sources/CodeEditTextView/Enums/CaptureName.swift create mode 100644 Sources/CodeEditTextView/Extensions/NSRange+/NSRange+InputEdit.swift create mode 100644 Sources/CodeEditTextView/Extensions/NSRange+/NSRange+NSTextRange.swift rename Sources/CodeEditTextView/Extensions/{String+NSRange.swift => NSRange+/NSRange+String.swift} (100%) create mode 100644 Sources/CodeEditTextView/Extensions/STTextView+/STTextView+VisibleRange.swift create mode 100644 Sources/CodeEditTextView/Highlighting/HighlightRange.swift create mode 100644 Sources/CodeEditTextView/Highlighting/Highlighter.swift delete mode 100644 Sources/CodeEditTextView/STTextViewController+CaptureNames.swift delete mode 100644 Sources/CodeEditTextView/STTextViewController+STTextViewDelegate.swift delete mode 100644 Sources/CodeEditTextView/STTextViewController+TreeSitter.swift create mode 100644 Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift diff --git a/Sources/CodeEditTextView/Enums/CaptureName.swift b/Sources/CodeEditTextView/Enums/CaptureName.swift new file mode 100644 index 000000000..8fc15c9b3 --- /dev/null +++ b/Sources/CodeEditTextView/Enums/CaptureName.swift @@ -0,0 +1,47 @@ +// +// STTextViewController+CaptureNames.swift +// CodeEditTextView +// +// Created by Lukas Pistrol on 16.08.22. +// + +/// A collection of possible capture names for `tree-sitter` with their respected raw values. +public enum CaptureName: String, CaseIterable { + case include + case constructor + case keyword + case boolean + case `repeat` + case conditional + case tag + case comment + case variable + case property + case function + case method + case number + case float + case string + case type + case parameter + case typeAlternate = "type_alternate" + case variableBuiltin = "variable.builtin" + case keywordReturn = "keyword.return" + case keywordFunction = "keyword.function" + + /// Returns a specific capture name case from a given string. + /// - Parameter string: A string to get the capture name from + /// - Returns: A `CaptureNames` case + static func fromString(_ string: String?) -> CaptureName? { + allCases.first { $0.rawValue == string } + } + + var alternate: CaptureName { + switch self { + case .type: + return .typeAlternate + default: + return self + } + } +} diff --git a/Sources/CodeEditTextView/Extensions/NSRange+/NSRange+InputEdit.swift b/Sources/CodeEditTextView/Extensions/NSRange+/NSRange+InputEdit.swift new file mode 100644 index 000000000..d86d22577 --- /dev/null +++ b/Sources/CodeEditTextView/Extensions/NSRange+/NSRange+InputEdit.swift @@ -0,0 +1,32 @@ +// +// NSRange+InputEdit.swift +// +// +// Created by Khan Winter on 9/12/22. +// + +import Foundation +import SwiftTreeSitter + +extension InputEdit { + init?(range: NSRange, delta: Int, oldEndPoint: Point) { + let startLocation = range.location + let newEndLocation = NSMaxRange(range) + delta + + if newEndLocation < 0 { + assertionFailure("Invalid range/delta") + return nil + } + + // TODO: - Ask why Neon only uses .zero for these + let startPoint: Point = .zero + let newEndPoint: Point = .zero + + self.init(startByte: UInt32(range.location * 2), + oldEndByte: UInt32(NSMaxRange(range) * 2), + newEndByte: UInt32(newEndLocation * 2), + startPoint: startPoint, + oldEndPoint: oldEndPoint, + newEndPoint: newEndPoint) + } +} diff --git a/Sources/CodeEditTextView/Extensions/NSRange+/NSRange+NSTextRange.swift b/Sources/CodeEditTextView/Extensions/NSRange+/NSRange+NSTextRange.swift new file mode 100644 index 000000000..a9a0a8737 --- /dev/null +++ b/Sources/CodeEditTextView/Extensions/NSRange+/NSRange+NSTextRange.swift @@ -0,0 +1,24 @@ +// +// NSRange+NSTextRange.swift +// +// +// Created by Khan Winter on 9/13/22. +// + +import AppKit + +public extension NSTextRange { + convenience init?(_ range: NSRange, provider: NSTextElementProvider) { + let docLocation = provider.documentRange.location + + guard let start = provider.location?(docLocation, offsetBy: range.location) else { + return nil + } + + guard let end = provider.location?(start, offsetBy: range.length) else { + return nil + } + + self.init(location: start, end: end) + } +} diff --git a/Sources/CodeEditTextView/Extensions/String+NSRange.swift b/Sources/CodeEditTextView/Extensions/NSRange+/NSRange+String.swift similarity index 100% rename from Sources/CodeEditTextView/Extensions/String+NSRange.swift rename to Sources/CodeEditTextView/Extensions/NSRange+/NSRange+String.swift diff --git a/Sources/CodeEditTextView/Extensions/STTextView+/STTextView+VisibleRange.swift b/Sources/CodeEditTextView/Extensions/STTextView+/STTextView+VisibleRange.swift new file mode 100644 index 000000000..a06b87cdb --- /dev/null +++ b/Sources/CodeEditTextView/Extensions/STTextView+/STTextView+VisibleRange.swift @@ -0,0 +1,31 @@ +// +// STTextView+VisibleRange.swift +// +// +// Created by Khan Winter on 9/12/22. +// + +import Foundation +import STTextView + +extension STTextView { + func textRange(for rect: CGRect) -> NSRange { + let length = self.textContentStorage.textStorage?.length ?? 0 + + guard let layoutManager = self.textContainer.layoutManager else { + return NSRange(0..<length) + } + let container = self.textContainer + + let origin = self.enclosingScrollView?.documentVisibleRect.origin ?? .zero + let glyphRange = layoutManager.glyphRange(forBoundingRect: rect, in: container) + + return layoutManager.characterRange(forGlyphRange: glyphRange, actualGlyphRange: nil) + } + + var visibleTextRange: NSRange { + get { + return textRange(for: visibleRect) + } + } +} diff --git a/Sources/CodeEditTextView/Highlighting/HighlightRange.swift b/Sources/CodeEditTextView/Highlighting/HighlightRange.swift new file mode 100644 index 000000000..7ba1bab67 --- /dev/null +++ b/Sources/CodeEditTextView/Highlighting/HighlightRange.swift @@ -0,0 +1,19 @@ +// +// HighlightRange.swift +// +// +// Created by Khan Winter on 9/14/22. +// + +import Foundation + +/// This class represents a range to highlight, as well as the capture name for syntax coloring. +class HighlightRange { + init(range: NSRange, capture: CaptureName?) { + self.range = range + self.capture = capture + } + + let range: NSRange + let capture: CaptureName? +} diff --git a/Sources/CodeEditTextView/Highlighting/Highlighter.swift b/Sources/CodeEditTextView/Highlighting/Highlighter.swift new file mode 100644 index 000000000..829c82cfd --- /dev/null +++ b/Sources/CodeEditTextView/Highlighting/Highlighter.swift @@ -0,0 +1,245 @@ +// +// Highlighter.swift +// +// +// Created by Khan Winter on 9/12/22. +// + +import Foundation +import AppKit +import STTextView +import SwiftTreeSitter + +/// Classes conforming to this protocol can provide attributes for text given a capture type. +public protocol ThemeAttributesProviding { + func attributesFor(_ capture: CaptureName?) -> [NSAttributedString.Key: Any] +} + +/// The `Highlighter` class handles efficiently highlighting the `STTextView` it's provided with. +/// It will listen for text and visibility changes, and highlight syntax as needed. +/// +/// One should rarely have to direcly modify or call methods on this class. Just keep it alive in +/// memory and it will listen for bounds changes, text changes, etc. However, to completely invalidate all +/// highlights use the ``invalidate()`` method to re-highlight all (visible) text, and the ``setLanguage`` +/// method to update the highlighter with a new language if needed. +class Highlighter: NSObject { + + // MARK: - Index Sets + + /// Any indexes that highlights have been requested for, but haven't been applied. + /// Indexes/ranges are added to this when highlights are requested and removed + /// after they are applied + private var pendingSet: IndexSet = .init() + + /// The set of valid indexes + private var validSet: IndexSet = .init() + + /// The range of the entire document + private var entireTextRange: Range<Int> { + get { + return 0..<(textView.textContentStorage.textStorage?.length ?? 0) + } + } + + /// The set of visible indexes in tht text view + lazy private var visibleSet: IndexSet = { + return IndexSet(integersIn: Range(textView.visibleTextRange)!) + }() + + // MARK: - UI + + /// The text view to highlight + private var textView: STTextView + private var theme: EditorTheme + private var attributeProvider: ThemeAttributesProviding! + + // MARK: - TreeSitter Client + + /// Calculates invalidated ranges given an edit. + private var treeSitterClient: TreeSitterClient + + // MARK: - Init + + /// Initializes the `Highlighter` + /// - Parameters: + /// - textView: The text view to highlight. + /// - treeSitterClient: The tree-sitter client to handle tree updates and highlight queries. + /// - theme: The theme to use for highlights. + init(textView: STTextView, + treeSitterClient: TreeSitterClient, + theme: EditorTheme, + attributeProvider: ThemeAttributesProviding) { + self.textView = textView + self.treeSitterClient = treeSitterClient + self.theme = theme + self.attributeProvider = attributeProvider + + super.init() + + treeSitterClient.setText(text: textView.string) + + guard textView.textContentStorage.textStorage != nil else { + assertionFailure("Text view does not have a textStorage") + return + } + + textView.textContentStorage.textStorage?.delegate = self + + if let scrollView = textView.enclosingScrollView { + NotificationCenter.default.addObserver(self, + selector: #selector(visibleTextChanged(_:)), + name: NSView.frameDidChangeNotification, + object: scrollView) + + NotificationCenter.default.addObserver(self, + selector: #selector(visibleTextChanged(_:)), + name: NSView.boundsDidChangeNotification, + object: scrollView.contentView) + } + } + + // MARK: - Public + + /// Invalidates all text in the textview. Useful for updating themes. + func invalidate() { + if !treeSitterClient.hasSetText { + treeSitterClient.setText(text: textView.string) + } + invalidate(range: entireTextRange) + } + + /// Sets the language and causes a re-highlight of the entire text. + /// - Parameter language: The language to update to. + func setLanguage(language: CodeLanguage) throws { + try treeSitterClient.setLanguage(codeLanguage: language, text: textView.string) + invalidate() + } + + deinit { + self.attributeProvider = nil + } +} + +// MARK: - Highlighting + +private extension Highlighter { + + /// Invalidates a given range and adds it to the queue to be highlighted. + /// - Parameter range: The range to invalidate. + func invalidate(range: NSRange) { + invalidate(range: Range(range)!) + } + + /// Invalidates a given range and adds it to the queue to be highlighted. + /// - Parameter range: The range to invalidate. + func invalidate(range: Range<Int>) { + let set = IndexSet(integersIn: range) + + if set.isEmpty { + return + } + + validSet.subtract(set) + + highlightNextRange() + } + + /// Begins highlighting any invalid ranges + func highlightNextRange() { + // If there aren't any more ranges to highlight, don't do anything, otherwise continue highlighting + // any available ranges. + guard let range = getNextRange() else { + return + } + + highlight(range: range) + + highlightNextRange() + } + + /// Highlights the given range + /// - Parameter range: The range to request highlights for. + func highlight(range nsRange: NSRange) { + let range = Range(nsRange)! + pendingSet.insert(integersIn: range) + + treeSitterClient.queryColorsFor(range: nsRange) { [weak self] highlightRanges in + guard let attributeProvider = self?.attributeProvider, + let textView = self?.textView else { return } + self?.pendingSet.remove(integersIn: range) + self?.validSet.formUnion(IndexSet(integersIn: range)) + highlightRanges.forEach { highlight in + textView.addAttributes(attributeProvider.attributesFor(highlight.capture), range: highlight.range, updateLayout: false) + } + } + } + + /// Gets the next `NSRange` to highlight based on the invalid set, visible set, and pending set. + /// - Returns: An `NSRange` to highlight if it could be fetched. + func getNextRange() -> NSRange? { + let set: IndexSet = IndexSet(integersIn: entireTextRange) // All text + .subtracting(validSet) // Subtract valid = Invalid set + .intersection(visibleSet) // Only visible indexes + .subtracting(pendingSet) // Don't include pending indexes + + guard let range = set.rangeView.map({ NSRange($0) }).first else { + return nil + } + + return range + } + +} + + +// MARK: - Visible Content Updates + +private extension Highlighter { + /// Updates the view to highlight newly visible text when the textview is scrolled or bounds change. + @objc func visibleTextChanged(_ notification: Notification) { + visibleSet = IndexSet(integersIn: Range(textView.visibleTextRange)!) + + // Any indices that are both *not* valid and in the visible text range should be invalidated + let newlyInvalidSet = visibleSet.subtracting(validSet) + + for range in newlyInvalidSet.rangeView.map({ NSRange($0) }) { + invalidate(range: range) + } + + } +} + +// MARK: - NSTextStorageDelegate + +extension Highlighter: NSTextStorageDelegate { + /// Processes an edited range in the text. Will query tree-sitter for any updated indices and re-highlight only the ranges that need it. + func textStorage(_ textStorage: NSTextStorage, + didProcessEditing editedMask: NSTextStorageEditActions, + range editedRange: NSRange, + changeInLength delta: Int) { + + // This method is called whenever attributes are updated, so to avoid re-highlighting the entire document + // each time an attribute is applied, we check to make sure this is in response to an edit. + guard editedMask.contains(.editedCharacters) else { + return + } + + let range = NSRange(location: editedRange.location, length: editedRange.length - delta) + + guard let edit = InputEdit(range: range, delta: delta, oldEndPoint: .zero) else { + return + } + + treeSitterClient.applyEdit(edit, + text: textStorage.string) { [weak self] invalidatedIndexSet in + let indexSet = invalidatedIndexSet + .union(IndexSet(integersIn: Range(editedRange)!)) + // Only invalidate indices that aren't visible. + .intersection(self?.visibleSet ?? .init()) + + for range in indexSet.rangeView { + self?.invalidate(range: NSRange(range)) + } + } + } +} diff --git a/Sources/CodeEditTextView/STTextViewController+CaptureNames.swift b/Sources/CodeEditTextView/STTextViewController+CaptureNames.swift deleted file mode 100644 index f193f48d6..000000000 --- a/Sources/CodeEditTextView/STTextViewController+CaptureNames.swift +++ /dev/null @@ -1,43 +0,0 @@ -// -// STTextViewController+CaptureNames.swift -// CodeEditTextView -// -// Created by Lukas Pistrol on 16.08.22. -// - -import Foundation - -internal extension STTextViewController { - - /// A collection of possible capture names for `tree-sitter` with their respected raw values. - enum CaptureNames: String, CaseIterable { - case include - case constructor - case keyword - case boolean - case `repeat` - case conditional - case tag - case comment - case variable - case property - case function - case method - case number - case float - case string - case type - case parameter - case typeAlternate = "type_alternate" - case variableBuiltin = "variable.builtin" - case keywordReturn = "keyword.return" - case keywordFunction = "keyword.function" - - /// Returns a specific capture name case from a given string. - /// - Parameter string: A string to get the capture name from - /// - Returns: A `CaptureNames` case - static func fromString(_ string: String?) -> CaptureNames? { - allCases.first { $0.rawValue == string } - } - } -} diff --git a/Sources/CodeEditTextView/STTextViewController+STTextViewDelegate.swift b/Sources/CodeEditTextView/STTextViewController+STTextViewDelegate.swift deleted file mode 100644 index 7d9668dd4..000000000 --- a/Sources/CodeEditTextView/STTextViewController+STTextViewDelegate.swift +++ /dev/null @@ -1,39 +0,0 @@ -// -// STTextViewController+STTextViewDelegate.swift -// CodeEditTextView -// -// Created by Lukas Pistrol on 28.05.22. -// - -import AppKit -import STTextView - -extension STTextViewController { - public func textDidChange(_ notification: Notification) { - print("Text did change") - } - - public func textView( - _ textView: STTextView, - shouldChangeTextIn affectedCharRange: NSTextRange, - replacementString: String? - ) -> Bool { - // Don't add '\t' characters - if replacementString == "\t" { - return false - } - return true - } - - public func textView( - _ textView: STTextView, - didChangeTextIn affectedCharRange: NSTextRange, - replacementString: String - ) { - textView.autocompleteBracketPairs(replacementString) - print("Did change text in \(affectedCharRange) | \(replacementString)") - highlight() - setStandardAttributes() - self.text.wrappedValue = textView.string - } -} diff --git a/Sources/CodeEditTextView/STTextViewController+TreeSitter.swift b/Sources/CodeEditTextView/STTextViewController+TreeSitter.swift deleted file mode 100644 index c8747b093..000000000 --- a/Sources/CodeEditTextView/STTextViewController+TreeSitter.swift +++ /dev/null @@ -1,96 +0,0 @@ -// -// STTextViewController+TreeSitter.swift -// CodeEditTextView -// -// Created by Lukas Pistrol on 28.05.22. -// - -import AppKit -import SwiftTreeSitter - -internal extension STTextViewController { - - /// Setup the `tree-sitter` parser and get queries. - func setupTreeSitter() { - DispatchQueue.global(qos: .userInitiated).async { - self.parser = Parser() - guard let lang = self.language.language else { return } - try? self.parser?.setLanguage(lang) - - let start = CFAbsoluteTimeGetCurrent() - self.query = TreeSitterModel.shared.query(for: self.language.id) - let end = CFAbsoluteTimeGetCurrent() - print("Fetching Query for \(self.language.id.rawValue): \(end-start) seconds") - DispatchQueue.main.async { - self.highlight() - } - } - } - - /// Execute queries and handle matches - func highlight() { - guard let parser = parser, - let text = textView?.string, - let tree = parser.parse(text), - let cursor = query?.execute(node: tree.rootNode!, in: tree) - else { return } - - if let expr = tree.rootNode?.sExpressionString, - expr.contains("ERROR") { return } - - while let match = cursor.next() { - // print("match: ", match) - self.highlightCaptures(match.captures) - self.highlightCaptures(for: match.predicates, in: match) - } - } - - /// Highlight query captures - func highlightCaptures(_ captures: [QueryCapture]) { - captures.forEach { capture in - textView?.addAttributes([ - .foregroundColor: colorForCapture(capture.name), - .font: NSFont.monospacedSystemFont(ofSize: font.pointSize, weight: .medium), - .baselineOffset: baselineOffset - ], range: capture.node.range) - } - } - - /// Highlight query captures for predicates - func highlightCaptures(for predicates: [Predicate], in match: QueryMatch) { - predicates.forEach { predicate in - predicate.captures(in: match).forEach { capture in - // print(capture.name, textView?.string[capture.node.range], predicate) - textView?.addAttributes( - [ - .foregroundColor: colorForCapture(capture.name?.appending("_alternate")), - .font: NSFont.monospacedSystemFont(ofSize: font.pointSize, weight: .medium), - .baselineOffset: baselineOffset - ], - range: capture.node.range - ) - } - } - } - - /// Get the color from ``theme`` for the specified capture name. - /// - Parameter capture: The capture name - /// - Returns: A `NSColor` - func colorForCapture(_ capture: String?) -> NSColor { - let captureName = CaptureNames.fromString(capture) - switch captureName { - case .include, .constructor, .keyword, .boolean, .variableBuiltin, - .keywordReturn, .keywordFunction, .repeat, .conditional, .tag: - return theme.keywords - case .comment: return theme.comments - case .variable, .property: return theme.variables - case .function, .method: return theme.variables - case .number, .float: return theme.numbers - case .string: return theme.strings - case .type: return theme.types - case .parameter: return theme.variables - case .typeAlternate: return theme.attributes - default: return theme.text - } - } -} diff --git a/Sources/CodeEditTextView/STTextViewController.swift b/Sources/CodeEditTextView/STTextViewController.swift index ea89f6915..23d38752c 100644 --- a/Sources/CodeEditTextView/STTextViewController.swift +++ b/Sources/CodeEditTextView/STTextViewController.swift @@ -11,7 +11,7 @@ import STTextView import SwiftTreeSitter /// A View Controller managing and displaying a `STTextView` -public class STTextViewController: NSViewController, STTextViewDelegate { +public class STTextViewController: NSViewController, STTextViewDelegate, ThemeAttributesProviding { internal var textView: STTextView! @@ -22,12 +22,13 @@ public class STTextViewController: NSViewController, STTextViewDelegate { /// The associated `CodeLanguage` public var language: CodeLanguage { didSet { - self.setupTreeSitter() + // TODO: Decide how to handle errors thrown here + try? highlighter?.setLanguage(language: language) }} /// The associated `Theme` used for highlighting. public var theme: EditorTheme { didSet { - highlight() + highlighter?.invalidate() }} /// The number of spaces to use for a `tab '\t'` character @@ -39,11 +40,10 @@ public class STTextViewController: NSViewController, STTextViewDelegate { /// The font to use in the `textView` public var font: NSFont - // MARK: Tree-Sitter + // MARK: - Highlighting - internal var parser: Parser? - internal var query: Query? - internal var tree: Tree? + internal var highlighter: Highlighter? + private var hasSetStandardAttributes: Bool = false // MARK: Init @@ -53,6 +53,7 @@ public class STTextViewController: NSViewController, STTextViewDelegate { self.font = font self.theme = theme self.tabWidth = tabWidth + super.init(nibName: nil, bundle: nil) } @@ -66,6 +67,10 @@ public class STTextViewController: NSViewController, STTextViewDelegate { let scrollView = STTextView.scrollableTextView() textView = scrollView.documentView as? STTextView + // By default this is always null but is required for a couple operations + // during highlighting so we make a new one manually. + textView.textContainer.replaceLayoutManager(NSLayoutManager()) + scrollView.translatesAutoresizingMaskIntoConstraints = false scrollView.hasVerticalScroller = true @@ -115,11 +120,28 @@ public class STTextViewController: NSViewController, STTextViewDelegate { self.keyUp(with: event) return event } + + setUpHighlighting() + } + + internal func setUpHighlighting() { + let textProvider: ResolvingQueryCursor.TextProvider = { [weak self] range, _ -> String? in + return self?.textView.textContentStorage.textStorage?.attributedSubstring(from: range).string + } + + let treeSitterClient = try! TreeSitterClient(codeLanguage: language, textProvider: textProvider) + self.highlighter = Highlighter(textView: textView, + treeSitterClient: treeSitterClient, + theme: theme, + attributeProvider: self) } public override func viewDidLoad() { super.viewDidLoad() - setupTreeSitter() + } + + public override func viewDidAppear() { + super.viewDidAppear() } // MARK: UI @@ -152,11 +174,20 @@ public class STTextViewController: NSViewController, STTextViewDelegate { /// Sets the standard attributes (`font`, `baselineOffset`) to the whole text internal func setStandardAttributes() { guard let textView = textView else { return } - textView.addAttributes([ + guard !hasSetStandardAttributes else { return } + hasSetStandardAttributes = true + textView.addAttributes(attributesFor(nil), range: .init(0..<textView.string.count)) + } + + /// Gets all attributes for the given capture including the line height, background color, and text color. + /// - Parameter capture: The capture to use for syntax highlighting. + /// - Returns: All attributes to be applied. + public func attributesFor(_ capture: CaptureName?) -> [NSAttributedString.Key: Any] { + return [ .font: font, - .foregroundColor: theme.text, + .foregroundColor: theme.colorFor(capture), .baselineOffset: baselineOffset - ], range: .init(0..<textView.string.count)) + ] } /// Calculated line height depending on ``STTextViewController/lineHeightMultiple`` @@ -189,4 +220,9 @@ public class STTextViewController: NSViewController, STTextViewDelegate { override public func keyUp(with event: NSEvent) { keyIsDown = false } + + deinit { + textView = nil + highlighter = nil + } } diff --git a/Sources/CodeEditTextView/Theme/EditorTheme.swift b/Sources/CodeEditTextView/Theme/EditorTheme.swift index 5d644e73c..ece94d550 100644 --- a/Sources/CodeEditTextView/Theme/EditorTheme.swift +++ b/Sources/CodeEditTextView/Theme/EditorTheme.swift @@ -62,4 +62,45 @@ public struct EditorTheme { self.characters = characters self.comments = comments } + + /// Get the color from ``theme`` for the specified capture name. + /// - Parameter capture: The capture name + /// - Returns: A `NSColor` + func colorFor(_ capture: CaptureName?) -> NSColor { + switch capture { + case .include, .constructor, .keyword, .boolean, .variableBuiltin, + .keywordReturn, .keywordFunction, .repeat, .conditional, .tag: + return keywords + case .comment: return comments + case .variable, .property: return variables + case .function, .method: return variables + case .number, .float: return numbers + case .string: return strings + case .type: return types + case .parameter: return variables + case .typeAlternate: return attributes + default: return text + } + } +} + +extension EditorTheme: Equatable { + public static func ==(lhs: EditorTheme, rhs: EditorTheme) -> Bool { + return lhs.text == rhs.text && + lhs.insertionPoint == rhs.insertionPoint && + lhs.invisibles == rhs.invisibles && + lhs.background == rhs.background && + lhs.lineHighlight == rhs.lineHighlight && + lhs.selection == rhs.selection && + lhs.keywords == rhs.keywords && + lhs.commands == rhs.commands && + lhs.types == rhs.types && + lhs.attributes == rhs.attributes && + lhs.variables == rhs.variables && + lhs.values == rhs.values && + lhs.numbers == rhs.numbers && + lhs.strings == rhs.strings && + lhs.characters == rhs.characters && + lhs.comments == rhs.comments + } } diff --git a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift new file mode 100644 index 000000000..8c44c365a --- /dev/null +++ b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift @@ -0,0 +1,175 @@ +// +// TreeSitterClient.swift +// +// +// Created by Khan Winter on 9/12/22. +// + +import Foundation +import SwiftTreeSitter + +/// `TreeSitterClient` is a class that manages applying edits for and querying captures for a syntax tree. +/// It handles queuing edits, processing them with the given text, and invalidating indices in the text for efficient +/// highlighting. +/// +/// Use the `init` method to set up the client initially. If text changes it should be able to be read through the +/// `textProvider` callback. You can optionally update the text manually using the `setText` method. +/// However, the `setText` method will re-compile the entire corpus so should be used sparingly. +final class TreeSitterClient { + internal var parser: Parser + internal var tree: Tree? + internal var languageQuery: Query? + + private var textProvider: ResolvingQueryCursor.TextProvider + + /// The queue to do tree-sitter work on for large edits/queries + private let queue: DispatchQueue = DispatchQueue(label: "CodeEdit.CodeEditTextView.TreeSitter", + qos: .userInteractive) + + /// Used to ensure safe use of the shared tree-sitter tree state in different sync/async contexts. + private let semaphore: DispatchSemaphore = DispatchSemaphore(value: 1) + + /// Initializes the `TreeSitterClient` with the given parameters. + /// - Parameters: + /// - codeLanguage: The language to set up the parser with. + /// - textProvider: The text provider callback to read any text. + public init(codeLanguage: CodeLanguage, textProvider: @escaping ResolvingQueryCursor.TextProvider) throws { + parser = Parser() + languageQuery = TreeSitterModel.shared.query(for: codeLanguage.id) + tree = nil + + self.textProvider = textProvider + + if let treeSitterLanguage = codeLanguage.language { + try parser.setLanguage(treeSitterLanguage) + } + } + + /// Set when the tree-sitter has text set. + public var hasSetText: Bool = false + + /// Reparses the tree-sitter tree using the given text. + /// - Parameter text: The text to parse. + public func setText(text: String) { + tree = self.parser.parse(text) + hasSetText = true + } + + /// Sets the language for the parser. Will cause a complete invalidation of the code, so use sparingly. + /// - Parameters: + /// - codeLanguage: The code language to use. + /// - text: The text to use to re-parse. + public func setLanguage(codeLanguage: CodeLanguage, text: String) throws { + parser = Parser() + languageQuery = TreeSitterModel.shared.query(for: codeLanguage.id) + + if let treeSitterLanguage = codeLanguage.language { + try parser.setLanguage(treeSitterLanguage) + } + + tree = self.parser.parse(text) + hasSetText = true + } + + /// Applies an edit to the code tree and calls the completion handler with any affected ranges. + /// - Parameters: + /// - edit: The edit to apply. + /// - text: The text content with the edit applied. + /// - completion: Called when affected ranges are found. + public func applyEdit(_ edit: InputEdit, text: String, completion: @escaping ((IndexSet) -> Void)) { + let readFunction = Parser.readFunction(for: text) + + let (oldTree, newTree) = self.calculateNewState(edit: edit, + text: text, + readBlock: readFunction) + + let effectedRanges = self.changedByteRanges(oldTree, rhs: newTree).map { $0.range } + + var rangeSet = IndexSet() + effectedRanges.forEach { range in + rangeSet.insert(integersIn: Range(range)!) + } + completion(rangeSet) + } + + /// Queries highlights for a given range. Will return on the main thread. + /// - Parameters: + /// - range: The range to query + /// - completion: Called with any highlights found in the query. + public func queryColorsFor(range: NSRange, completion: @escaping (([HighlightRange]) -> Void)) { + self.semaphore.wait() + guard let tree = self.tree?.copy() else { + self.semaphore.signal() + completion([]) + return + } + self.semaphore.signal() + + guard let rootNode = tree.rootNode else { + completion([]) + return + } + + // This needs to be on the main thread since we're going to use the `textProvider` in + // the `highlightsFromCursor` method, which uses the textView's text storage. + guard let cursor = self.languageQuery?.execute(node: rootNode, in: tree) else { + completion([]) + return + } + cursor.setRange(range) + let highlights = self.highlightsFromCursor(cursor: ResolvingQueryCursor(cursor: cursor)) + completion(highlights) + } + + /// Resolves a query cursor to the highlight ranges it contains. + /// **Must be called on the main thread** + /// - Parameter cursor: The cursor to resolve. + /// - Returns: Any highlight ranges contained in the cursor. + private func highlightsFromCursor(cursor: ResolvingQueryCursor) -> [HighlightRange] { + cursor.prepare(with: self.textProvider) + return cursor + .flatMap { $0.captures } + .map { HighlightRange(range: $0.range, capture: CaptureName(rawValue: $0.name ?? "")) } + } +} + +extension TreeSitterClient { + /// Applies the edit to the current `tree` and returns the old tree and a copy of the current tree with the + /// processed edit. + /// - Parameter edit: The edit to apply. + /// - Returns: (The old state, the new state). + private func calculateNewState(edit: InputEdit, text: String, readBlock: @escaping Parser.ReadBlock) -> (Tree?, Tree?) { + guard let oldTree = self.tree else { + self.tree = self.parser.parse(text) + return (nil, self.tree) + } + self.semaphore.wait() + + // Apply the edit to the old tree + oldTree.edit(edit) + + self.tree = self.parser.parse(tree: oldTree, readBlock: readBlock) + + self.semaphore.signal() + + return (oldTree.copy(), self.tree?.copy()) + } + + /// Calculates the changed byte ranges between two trees. + /// - Parameters: + /// - lhs: The first (older) tree. + /// - rhs: The second (newer) tree. + /// - Returns: Any changed ranges. + private func changedByteRanges(_ lhs: Tree?, rhs: Tree?) -> [Range<UInt32>] { + switch (lhs, rhs) { + case (let t1?, let t2?): + return t1.changedRanges(from: t2).map({ $0.bytes }) + case (nil, let t2?): + let range = t2.rootNode?.byteRange + + return range.flatMap({ [$0] }) ?? [] + case (_, nil): + return [] + } + } +} From e7b7d851e5720a9ce6a0052eb2a70165e38b7c63 Mon Sep 17 00:00:00 2001 From: Khan Winter <35942988+thecoolwinter@users.noreply.github.com> Date: Fri, 30 Sep 2022 15:32:58 -0500 Subject: [PATCH 2/4] Faster Highlighting, Fix Linter - Replaces `textView.addAttributes` by modifying the `textContentStorage` directly. This led to a 10x speedup with large files*. - Makes treeSitterClient optional on the Highlighter object for the case where tree sitter can't be initialized, doesn't have language support, or something else. - Fixes SwiftLint errors. * This speed still feels slow and glitchy, and should be revisited in a future PR. --- .../STTextView+/STTextView+VisibleRange.swift | 4 +- .../Highlighting/Highlighter.swift | 37 +++++++++++-------- .../STTextViewController.swift | 3 +- .../CodeEditTextView/Theme/EditorTheme.swift | 2 +- .../TreeSitter/TreeSitterClient.swift | 4 +- 5 files changed, 29 insertions(+), 21 deletions(-) diff --git a/Sources/CodeEditTextView/Extensions/STTextView+/STTextView+VisibleRange.swift b/Sources/CodeEditTextView/Extensions/STTextView+/STTextView+VisibleRange.swift index a06b87cdb..085a6034a 100644 --- a/Sources/CodeEditTextView/Extensions/STTextView+/STTextView+VisibleRange.swift +++ b/Sources/CodeEditTextView/Extensions/STTextView+/STTextView+VisibleRange.swift @@ -24,8 +24,6 @@ extension STTextView { } var visibleTextRange: NSRange { - get { - return textRange(for: visibleRect) - } + return textRange(for: visibleRect) } } diff --git a/Sources/CodeEditTextView/Highlighting/Highlighter.swift b/Sources/CodeEditTextView/Highlighting/Highlighter.swift index 829c82cfd..9757cc548 100644 --- a/Sources/CodeEditTextView/Highlighting/Highlighter.swift +++ b/Sources/CodeEditTextView/Highlighting/Highlighter.swift @@ -36,9 +36,7 @@ class Highlighter: NSObject { /// The range of the entire document private var entireTextRange: Range<Int> { - get { - return 0..<(textView.textContentStorage.textStorage?.length ?? 0) - } + return 0..<(textView.textContentStorage.textStorage?.length ?? 0) } /// The set of visible indexes in tht text view @@ -56,7 +54,7 @@ class Highlighter: NSObject { // MARK: - TreeSitter Client /// Calculates invalidated ranges given an edit. - private var treeSitterClient: TreeSitterClient + private var treeSitterClient: TreeSitterClient? // MARK: - Init @@ -66,7 +64,7 @@ class Highlighter: NSObject { /// - treeSitterClient: The tree-sitter client to handle tree updates and highlight queries. /// - theme: The theme to use for highlights. init(textView: STTextView, - treeSitterClient: TreeSitterClient, + treeSitterClient: TreeSitterClient?, theme: EditorTheme, attributeProvider: ThemeAttributesProviding) { self.textView = textView @@ -76,7 +74,7 @@ class Highlighter: NSObject { super.init() - treeSitterClient.setText(text: textView.string) + treeSitterClient?.setText(text: textView.string) guard textView.textContentStorage.textStorage != nil else { assertionFailure("Text view does not have a textStorage") @@ -102,8 +100,8 @@ class Highlighter: NSObject { /// Invalidates all text in the textview. Useful for updating themes. func invalidate() { - if !treeSitterClient.hasSetText { - treeSitterClient.setText(text: textView.string) + if !(treeSitterClient?.hasSetText ?? true) { + treeSitterClient?.setText(text: textView.string) } invalidate(range: entireTextRange) } @@ -111,7 +109,7 @@ class Highlighter: NSObject { /// Sets the language and causes a re-highlight of the entire text. /// - Parameter language: The language to update to. func setLanguage(language: CodeLanguage) throws { - try treeSitterClient.setLanguage(codeLanguage: language, text: textView.string) + try treeSitterClient?.setLanguage(codeLanguage: language, text: textView.string) invalidate() } @@ -163,14 +161,23 @@ private extension Highlighter { let range = Range(nsRange)! pendingSet.insert(integersIn: range) - treeSitterClient.queryColorsFor(range: nsRange) { [weak self] highlightRanges in + treeSitterClient?.queryColorsFor(range: nsRange) { [weak self] highlightRanges in guard let attributeProvider = self?.attributeProvider, let textView = self?.textView else { return } self?.pendingSet.remove(integersIn: range) self?.validSet.formUnion(IndexSet(integersIn: range)) - highlightRanges.forEach { highlight in - textView.addAttributes(attributeProvider.attributesFor(highlight.capture), range: highlight.range, updateLayout: false) + if !(self?.visibleSet ?? .init()).contains(integersIn: range) { + return + } + + textView.textContentStorage.textStorage?.beginEditing() + for highlight in highlightRanges { + textView.textContentStorage.textStorage?.setAttributes( + attributeProvider.attributesFor(highlight.capture), + range: highlight.range + ) } + textView.textContentStorage.textStorage?.endEditing() } } @@ -191,7 +198,6 @@ private extension Highlighter { } - // MARK: - Visible Content Updates private extension Highlighter { @@ -212,7 +218,8 @@ private extension Highlighter { // MARK: - NSTextStorageDelegate extension Highlighter: NSTextStorageDelegate { - /// Processes an edited range in the text. Will query tree-sitter for any updated indices and re-highlight only the ranges that need it. + /// Processes an edited range in the text. + /// Will query tree-sitter for any updated indices and re-highlight only the ranges that need it. func textStorage(_ textStorage: NSTextStorage, didProcessEditing editedMask: NSTextStorageEditActions, range editedRange: NSRange, @@ -230,7 +237,7 @@ extension Highlighter: NSTextStorageDelegate { return } - treeSitterClient.applyEdit(edit, + treeSitterClient?.applyEdit(edit, text: textStorage.string) { [weak self] invalidatedIndexSet in let indexSet = invalidatedIndexSet .union(IndexSet(integersIn: Range(editedRange)!)) diff --git a/Sources/CodeEditTextView/STTextViewController.swift b/Sources/CodeEditTextView/STTextViewController.swift index 23d38752c..6a05b1cd5 100644 --- a/Sources/CodeEditTextView/STTextViewController.swift +++ b/Sources/CodeEditTextView/STTextViewController.swift @@ -63,6 +63,7 @@ public class STTextViewController: NSViewController, STTextViewDelegate, ThemeAt // MARK: VC Lifecycle + // swiftlint:disable function_body_length public override func loadView() { let scrollView = STTextView.scrollableTextView() textView = scrollView.documentView as? STTextView @@ -129,7 +130,7 @@ public class STTextViewController: NSViewController, STTextViewDelegate, ThemeAt return self?.textView.textContentStorage.textStorage?.attributedSubstring(from: range).string } - let treeSitterClient = try! TreeSitterClient(codeLanguage: language, textProvider: textProvider) + let treeSitterClient = try? TreeSitterClient(codeLanguage: language, textProvider: textProvider) self.highlighter = Highlighter(textView: textView, treeSitterClient: treeSitterClient, theme: theme, diff --git a/Sources/CodeEditTextView/Theme/EditorTheme.swift b/Sources/CodeEditTextView/Theme/EditorTheme.swift index ece94d550..39e1be9ce 100644 --- a/Sources/CodeEditTextView/Theme/EditorTheme.swift +++ b/Sources/CodeEditTextView/Theme/EditorTheme.swift @@ -85,7 +85,7 @@ public struct EditorTheme { } extension EditorTheme: Equatable { - public static func ==(lhs: EditorTheme, rhs: EditorTheme) -> Bool { + public static func == (lhs: EditorTheme, rhs: EditorTheme) -> Bool { return lhs.text == rhs.text && lhs.insertionPoint == rhs.insertionPoint && lhs.invisibles == rhs.invisibles && diff --git a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift index 8c44c365a..008ee5781 100644 --- a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift +++ b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift @@ -138,7 +138,9 @@ extension TreeSitterClient { /// processed edit. /// - Parameter edit: The edit to apply. /// - Returns: (The old state, the new state). - private func calculateNewState(edit: InputEdit, text: String, readBlock: @escaping Parser.ReadBlock) -> (Tree?, Tree?) { + private func calculateNewState(edit: InputEdit, + text: String, + readBlock: @escaping Parser.ReadBlock) -> (Tree?, Tree?) { guard let oldTree = self.tree else { self.tree = self.parser.parse(text) return (nil, self.tree) From 39f07496e8f91494b7bb75bfaccbdb6a9ae02070 Mon Sep 17 00:00:00 2001 From: Khan Winter <35942988+thecoolwinter@users.noreply.github.com> Date: Sat, 1 Oct 2022 20:25:42 -0500 Subject: [PATCH 3/4] Invalidate layout on highlight --- .../Highlighting/Highlighter.swift | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/Sources/CodeEditTextView/Highlighting/Highlighter.swift b/Sources/CodeEditTextView/Highlighting/Highlighter.swift index 9757cc548..27dd6ef48 100644 --- a/Sources/CodeEditTextView/Highlighting/Highlighter.swift +++ b/Sources/CodeEditTextView/Highlighting/Highlighter.swift @@ -164,12 +164,23 @@ private extension Highlighter { treeSitterClient?.queryColorsFor(range: nsRange) { [weak self] highlightRanges in guard let attributeProvider = self?.attributeProvider, let textView = self?.textView else { return } + + // Mark these indices as not pending and valid self?.pendingSet.remove(integersIn: range) self?.validSet.formUnion(IndexSet(integersIn: range)) + + // If this range does not exist in the visible set, we can exit. if !(self?.visibleSet ?? .init()).contains(integersIn: range) { return } + // Try to create a text range for invalidating. If this fails we fail silently + guard let textContentManager = textView.textLayoutManager.textContentManager, + let textRange = NSTextRange(nsRange, provider: textContentManager) else { + return + } + + // Loop through each highlight and modify the textStorage accordingly. textView.textContentStorage.textStorage?.beginEditing() for highlight in highlightRanges { textView.textContentStorage.textStorage?.setAttributes( @@ -178,6 +189,10 @@ private extension Highlighter { ) } textView.textContentStorage.textStorage?.endEditing() + + // After applying edits to the text storage we need to invalidate the layout + // of the highlighted text. + textView.textLayoutManager.invalidateLayout(for: textRange) } } @@ -211,7 +226,6 @@ private extension Highlighter { for range in newlyInvalidSet.rangeView.map({ NSRange($0) }) { invalidate(range: range) } - } } From 712cb58f79079a67fb12d25b8d9d02e3dd6ad887 Mon Sep 17 00:00:00 2001 From: Khan Winter <35942988+thecoolwinter@users.noreply.github.com> Date: Sat, 1 Oct 2022 20:35:01 -0500 Subject: [PATCH 4/4] Fix tests, remove warnings --- .../Extensions/NSRange+/NSRange+InputEdit.swift | 1 - .../STTextView+/STTextView+VisibleRange.swift | 1 - .../STTextViewControllerTests.swift | 17 +++++++++-------- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/Sources/CodeEditTextView/Extensions/NSRange+/NSRange+InputEdit.swift b/Sources/CodeEditTextView/Extensions/NSRange+/NSRange+InputEdit.swift index d86d22577..9994a92df 100644 --- a/Sources/CodeEditTextView/Extensions/NSRange+/NSRange+InputEdit.swift +++ b/Sources/CodeEditTextView/Extensions/NSRange+/NSRange+InputEdit.swift @@ -10,7 +10,6 @@ import SwiftTreeSitter extension InputEdit { init?(range: NSRange, delta: Int, oldEndPoint: Point) { - let startLocation = range.location let newEndLocation = NSMaxRange(range) + delta if newEndLocation < 0 { diff --git a/Sources/CodeEditTextView/Extensions/STTextView+/STTextView+VisibleRange.swift b/Sources/CodeEditTextView/Extensions/STTextView+/STTextView+VisibleRange.swift index 085a6034a..d6bb88bd6 100644 --- a/Sources/CodeEditTextView/Extensions/STTextView+/STTextView+VisibleRange.swift +++ b/Sources/CodeEditTextView/Extensions/STTextView+/STTextView+VisibleRange.swift @@ -17,7 +17,6 @@ extension STTextView { } let container = self.textContainer - let origin = self.enclosingScrollView?.documentVisibleRect.origin ?? .zero let glyphRange = layoutManager.glyphRange(forBoundingRect: rect, in: container) return layoutManager.characterRange(forGlyphRange: glyphRange, actualGlyphRange: nil) diff --git a/Tests/CodeEditTextViewTests/STTextViewControllerTests.swift b/Tests/CodeEditTextViewTests/STTextViewControllerTests.swift index a6b7f3014..8da553545 100644 --- a/Tests/CodeEditTextViewTests/STTextViewControllerTests.swift +++ b/Tests/CodeEditTextViewTests/STTextViewControllerTests.swift @@ -1,6 +1,7 @@ import XCTest @testable import CodeEditTextView import SwiftTreeSitter +import AppKit final class STTextViewControllerTests: XCTestCase { @@ -38,25 +39,25 @@ final class STTextViewControllerTests: XCTestCase { func test_captureNames() throws { // test for "keyword" let captureName1 = "keyword" - let color1 = controller.colorForCapture(captureName1) - XCTAssertEqual(color1, .systemPink) + let color1 = controller.attributesFor(CaptureName(rawValue: captureName1))[.foregroundColor] as? NSColor + XCTAssertEqual(color1, NSColor.systemPink) // test for "comment" let captureName2 = "comment" - let color2 = controller.colorForCapture(captureName2) - XCTAssertEqual(color2, .systemGreen) + let color2 = controller.attributesFor(CaptureName(rawValue: captureName2))[.foregroundColor] as? NSColor + XCTAssertEqual(color2, NSColor.systemGreen) /* ... additional tests here ... */ // test for empty case let captureName3 = "" - let color3 = controller.colorForCapture(captureName3) - XCTAssertEqual(color3, .textColor) + let color3 = controller.attributesFor(CaptureName(rawValue: captureName3))[.foregroundColor] as? NSColor + XCTAssertEqual(color3, NSColor.textColor) // test for random case let captureName4 = "abc123" - let color4 = controller.colorForCapture(captureName4) - XCTAssertEqual(color4, .textColor) + let color4 = controller.attributesFor(CaptureName(rawValue: captureName4))[.foregroundColor] as? NSColor + XCTAssertEqual(color4, NSColor.textColor) } }