From 51d95ad61bfe350e8cc24096da5b1f92b5335a2d Mon Sep 17 00:00:00 2001 From: Khan Winter <35942988+thecoolwinter@users.noreply.github.com> Date: Mon, 22 May 2023 12:34:58 -0500 Subject: [PATCH 01/10] Initial Implementation --- .../HighlighterTextView+createReadBlock.swift | 25 ++ .../Extensions/Parser+createTree.swift | 20 ++ .../STTextView+/STTextView+VisibleRange.swift | 4 +- .../Highlighting/Highlighter.swift | 30 +- .../Highlighting/HighlighterTextView.swift | 2 +- .../TreeSitter/TSLanguageLayer.swift | 140 ++++++++ .../TreeSitter/TreeSitterClient+Edit.swift | 225 ++++++------- .../TreeSitterClient+Highlight.swift | 107 ++++--- .../TreeSitterClient+LanguageLayer.swift | 54 ---- .../TreeSitter/TreeSitterClient.swift | 298 +++++++----------- .../TreeSitter/TreeSitterState.swift | 255 +++++++++++++++ 11 files changed, 733 insertions(+), 427 deletions(-) create mode 100644 Sources/CodeEditTextView/Extensions/HighlighterTextView+createReadBlock.swift create mode 100644 Sources/CodeEditTextView/Extensions/Parser+createTree.swift create mode 100644 Sources/CodeEditTextView/TreeSitter/TSLanguageLayer.swift delete mode 100644 Sources/CodeEditTextView/TreeSitter/TreeSitterClient+LanguageLayer.swift create mode 100644 Sources/CodeEditTextView/TreeSitter/TreeSitterState.swift diff --git a/Sources/CodeEditTextView/Extensions/HighlighterTextView+createReadBlock.swift b/Sources/CodeEditTextView/Extensions/HighlighterTextView+createReadBlock.swift new file mode 100644 index 000000000..54efa8161 --- /dev/null +++ b/Sources/CodeEditTextView/Extensions/HighlighterTextView+createReadBlock.swift @@ -0,0 +1,25 @@ +// +// HighlighterTextView+createReadBlock.swift +// +// +// Created by Khan Winter on 5/20/23. +// + +import Foundation +import SwiftTreeSitter + +extension HighlighterTextView { + func createReadBlock() -> Parser.ReadBlock { + return { byteOffset, _ in + let limit = self.documentRange.length + let location = byteOffset / 2 + let end = min(location + (1024), limit) + if location > end { + // Ignore and return nothing, tree-sitter's internal tree can be incorrect in some situations. + return nil + } + let range = NSRange(location.. Tree? { + return parse(tree: nil, readBlock: readBlock) + } +} diff --git a/Sources/CodeEditTextView/Extensions/STTextView+/STTextView+VisibleRange.swift b/Sources/CodeEditTextView/Extensions/STTextView+/STTextView+VisibleRange.swift index 64588f433..722017cf5 100644 --- a/Sources/CodeEditTextView/Extensions/STTextView+/STTextView+VisibleRange.swift +++ b/Sources/CodeEditTextView/Extensions/STTextView+/STTextView+VisibleRange.swift @@ -21,7 +21,7 @@ extension STTextView { // Get visible rect guard let bounds = enclosingScrollView?.documentVisibleRect else { - return textLayoutManager.documentRange.nsRange(using: textContentStorage) + return nil } // Calculate min & max points w/ a small amount of padding vertically. @@ -33,7 +33,7 @@ extension STTextView { // Get text fragments for both the min and max points guard let start = textLayoutManager.textLayoutFragment(for: minPoint)?.rangeInElement.location, let end = textLayoutManager.textLayoutFragment(for: maxPoint)?.rangeInElement.endLocation else { - return textLayoutManager.documentRange.nsRange(using: textContentStorage) + return nil } guard start.compare(end) != .orderedDescending else { return NSTextRange(location: end, end: start)?.nsRange(using: textContentStorage) diff --git a/Sources/CodeEditTextView/Highlighting/Highlighter.swift b/Sources/CodeEditTextView/Highlighting/Highlighter.swift index c31d67a80..157fc353a 100644 --- a/Sources/CodeEditTextView/Highlighting/Highlighter.swift +++ b/Sources/CodeEditTextView/Highlighting/Highlighter.swift @@ -167,19 +167,18 @@ private extension Highlighter { func highlight(range rangeToHighlight: NSRange) { pendingSet.insert(integersIn: rangeToHighlight) + print("--Querying Range", rangeToHighlight, visibleSet.rangeView.map { $0 }) highlightProvider?.queryHighlightsFor(textView: self.textView, range: rangeToHighlight) { [weak self] highlightRanges in + print("--Received Query", rangeToHighlight, self!.visibleSet.rangeView.map { $0 }) guard let attributeProvider = self?.attributeProvider, let textView = self?.textView else { return } - // Mark these indices as not pending and valid self?.pendingSet.remove(integersIn: rangeToHighlight) - self?.validSet.formUnion(IndexSet(integersIn: rangeToHighlight)) - - // If this range does not exist in the visible set, we can exit. - if !(self?.visibleSet ?? .init()).contains(integersIn: rangeToHighlight) { + guard self?.visibleSet.intersects(integersIn: rangeToHighlight) ?? false else { return } + self?.validSet.formUnion(IndexSet(integersIn: rangeToHighlight)) // Try to create a text range for invalidating. If this fails we fail silently guard let textContentManager = textView.textLayoutManager.textContentManager, @@ -251,7 +250,9 @@ private extension Highlighter { private extension Highlighter { /// Updates the view to highlight newly visible text when the textview is scrolled or bounds change. @objc func visibleTextChanged(_ notification: Notification) { - visibleSet = IndexSet(integersIn: textView.visibleTextRange ?? NSRange()) + if let newVisibleRange = textView.visibleTextRange { + visibleSet = IndexSet(integersIn: newVisibleRange) + } // Any indices that are both *not* valid and in the visible text range should be invalidated let newlyInvalidSet = visibleSet.subtracting(validSet) @@ -281,10 +282,21 @@ extension Highlighter: NSTextStorageDelegate { if delta > 0 { visibleSet.insert(range: editedRange) } + var info = mach_timebase_info() + guard mach_timebase_info(&info) == KERN_SUCCESS else { return } + + let start = mach_absolute_time() highlightProvider?.applyEdit(textView: self.textView, range: range, delta: delta) { [weak self] invalidatedIndexSet in + let end = mach_absolute_time() + + let elapsed = end - start + + let nanos = elapsed * UInt64(info.numer) / UInt64(info.denom) + print("Receiving Results", TimeInterval(nanos) / TimeInterval(NSEC_PER_MSEC)) + let indexSet = invalidatedIndexSet .union(IndexSet(integersIn: editedRange)) // Only invalidate indices that are visible. @@ -294,5 +306,11 @@ extension Highlighter: NSTextStorageDelegate { self?.invalidate(range: NSRange(range)) } } + let end = mach_absolute_time() + + let elapsed = end - start + + let nanos = elapsed * UInt64(info.numer) / UInt64(info.denom) + print("Returning From Function", TimeInterval(nanos) / TimeInterval(NSEC_PER_MSEC)) } } diff --git a/Sources/CodeEditTextView/Highlighting/HighlighterTextView.swift b/Sources/CodeEditTextView/Highlighting/HighlighterTextView.swift index 7286b320c..a79755375 100644 --- a/Sources/CodeEditTextView/Highlighting/HighlighterTextView.swift +++ b/Sources/CodeEditTextView/Highlighting/HighlighterTextView.swift @@ -10,7 +10,7 @@ import AppKit import STTextView /// The object `HighlightProviding` objects are given when asked for highlights. -public protocol HighlighterTextView { +public protocol HighlighterTextView: AnyObject { /// The entire range of the document. var documentRange: NSRange { get } /// A substring for the requested range. diff --git a/Sources/CodeEditTextView/TreeSitter/TSLanguageLayer.swift b/Sources/CodeEditTextView/TreeSitter/TSLanguageLayer.swift new file mode 100644 index 000000000..9bdb4b552 --- /dev/null +++ b/Sources/CodeEditTextView/TreeSitter/TSLanguageLayer.swift @@ -0,0 +1,140 @@ +// +// TreeSitterClient+LanguageLayer.swift +// +// +// Created by Khan Winter on 3/8/23. +// + +import Foundation +import CodeEditLanguages +import SwiftTreeSitter + +class TSLanguageLayer: Hashable { + /// Initialize a language layer + /// - Parameters: + /// - id: The ID of the layer. + /// - parser: A parser to use for the layer. + /// - supportsInjections: Set to true when the langauge supports the `injections` query. + /// - tree: The tree-sitter tree generated while editing/parsing a document. + /// - languageQuery: The language query used for fetching the associated `queries.scm` file + /// - ranges: All ranges this layer acts on. Must be kept in order and w/o overlap. + init( + id: TreeSitterLanguage, + parser: Parser, + supportsInjections: Bool, + tree: Tree? = nil, + languageQuery: Query? = nil, + ranges: [NSRange] + ) { + self.id = id + self.parser = parser + self.supportsInjections = supportsInjections + self.tree = tree + self.languageQuery = languageQuery + self.ranges = ranges + + self.parser.timeout = TreeSitterClient.Constants.parserTimeout + } + + let id: TreeSitterLanguage + let parser: Parser + let supportsInjections: Bool + var tree: Tree? + var languageQuery: Query? + var ranges: [NSRange] + + static func == (lhs: TSLanguageLayer, rhs: TSLanguageLayer) -> Bool { + return lhs.id == rhs.id && lhs.ranges == rhs.ranges + } + + func hash(into hasher: inout Hasher) { + hasher.combine(id) + hasher.combine(ranges) + } + + /// Calculates a series of ranges that have been invalidated by a given edit. + /// - Parameters: + /// - textView: The text view to use for text. + /// - edit: The edit to act on. + /// - timeout: The maximum time interval the parser can run before halting. + /// - readBlock: A callback for fetching blocks of text. + /// - Returns: An array of distinct `NSRanges` that need to be re-highlighted. + func findChangedByteRanges( + textView: HighlighterTextView, + edit: InputEdit, + timeout: TimeInterval?, + readBlock: @escaping Parser.ReadBlock + ) throws -> [NSRange] { + parser.timeout = timeout ?? 0 + + let (oldTree, newTree) = calculateNewState( + tree: self.tree, + parser: self.parser, + edit: edit, + readBlock: readBlock + ) + + if oldTree == nil && newTree == nil { + // There was no existing tree, make a new one and return all indexes. + tree = parser.createTree(readBlock: readBlock) + return [NSRange(textView.documentRange.intRange)] + } else if oldTree != nil && newTree == nil { + // The parser timed out, + throw Error.parserTimeout + } + + let ranges = changedByteRanges(oldTree, rhs: newTree).map { $0.range } + + tree = newTree + + return ranges + } + + /// Applies the edit to the current `tree` and returns the old tree and a copy of the current tree with the + /// processed edit. + /// - Parameters: + /// - tree: The tree before an edit used to parse the new tree. + /// - parser: The parser used to parse the new tree. + /// - edit: The edit to apply. + /// - readBlock: The block to use to read text. + /// - Returns: (The old state, the new state). + internal func calculateNewState( + tree: Tree?, + parser: Parser, + edit: InputEdit, + readBlock: @escaping Parser.ReadBlock + ) -> (Tree?, Tree?) { + guard let oldTree = tree else { + return (nil, nil) + } + + // Apply the edit to the old tree + oldTree.edit(edit) + + let newTree = parser.parse(tree: oldTree, readBlock: readBlock) + + return (oldTree.copy(), newTree) + } + + /// Calculates the changed byte ranges between two trees. + /// - Parameters: + /// - lhs: The first (older) tree. + /// - rhs: The second (newer) tree. + /// - Returns: Any changed ranges. + internal func changedByteRanges(_ lhs: Tree?, rhs: Tree?) -> [Range] { + switch (lhs, rhs) { + case (let tree1?, let tree2?): + return tree1.changedRanges(from: tree2).map({ $0.bytes }) + case (nil, let tree2?): + let range = tree2.rootNode?.byteRange + + return range.flatMap({ [$0] }) ?? [] + case (_, nil): + return [] + } + } + + enum Error: Swift.Error { + case parserTimeout + } +} diff --git a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+Edit.swift b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+Edit.swift index 871d59cf6..10dc8bc6b 100644 --- a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+Edit.swift +++ b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+Edit.swift @@ -10,154 +10,111 @@ import SwiftTreeSitter import CodeEditLanguages extension TreeSitterClient { - - /// Calculates a series of ranges that have been invalidated by a given edit. - /// - Parameters: - /// - textView: The text view to use for text. - /// - edit: The edit to act on. - /// - language: The language to use. - /// - readBlock: A callback for fetching blocks of text. - /// - Returns: An array of distinct `NSRanges` that need to be re-highlighted. - func findChangedByteRanges( - textView: HighlighterTextView, - edit: InputEdit, - layer: LanguageLayer, - readBlock: @escaping Parser.ReadBlock - ) -> [NSRange] { - let (oldTree, newTree) = calculateNewState( - tree: layer.tree, - parser: layer.parser, - edit: edit, - readBlock: readBlock - ) - if oldTree == nil && newTree == nil { - // There was no existing tree, make a new one and return all indexes. - layer.tree = createTree(parser: layer.parser, readBlock: readBlock) - return [NSRange(textView.documentRange.intRange)] - } - - let ranges = changedByteRanges(oldTree, rhs: newTree).map { $0.range } - - layer.tree = newTree - - return ranges - } - - /// Applies the edit to the current `tree` and returns the old tree and a copy of the current tree with the - /// processed edit. - /// - Parameters: - /// - tree: The tree before an edit used to parse the new tree. - /// - parser: The parser used to parse the new tree. - /// - edit: The edit to apply. - /// - readBlock: The block to use to read text. - /// - Returns: (The old state, the new state). - internal func calculateNewState( - tree: Tree?, - parser: Parser, - edit: InputEdit, - readBlock: @escaping Parser.ReadBlock - ) -> (Tree?, Tree?) { - guard let oldTree = tree else { - return (nil, nil) - } - semaphore.wait() - - // Apply the edit to the old tree - oldTree.edit(edit) - - let newTree = parser.parse(tree: oldTree, readBlock: readBlock) - - semaphore.signal() - - return (oldTree.copy(), newTree) - } - - /// Calculates the changed byte ranges between two trees. - /// - Parameters: - /// - lhs: The first (older) tree. - /// - rhs: The second (newer) tree. - /// - Returns: Any changed ranges. - internal func changedByteRanges(_ lhs: Tree?, rhs: Tree?) -> [Range] { - switch (lhs, rhs) { - case (let tree1?, let tree2?): - return tree1.changedRanges(from: tree2).map({ $0.bytes }) - case (nil, let tree2?): - let range = tree2.rootNode?.byteRange - - return range.flatMap({ [$0] }) ?? [] - case (_, nil): - return [] + /// A helper class for passing edit state from synchronous to asynchronous contexts. + class EditState { + var edit: InputEdit + var rangeSet: IndexSet + var layerSet: Set + var touchedLayers: Set + var completion: ((IndexSet) -> Void) + + init( + edit: InputEdit, + minimumCapacity: Int = 0, + completion: @escaping (IndexSet) -> Void + ) { + self.edit = edit + self.rangeSet = IndexSet() + self.layerSet = Set(minimumCapacity: minimumCapacity) + self.touchedLayers = Set(minimumCapacity: minimumCapacity) + self.completion = completion } } - /// Performs an injections query on the given language layer. - /// Updates any existing layers with new ranges and adds new layers if needed. + /// Applies the given edit to the current state and calls the editState's completion handler. /// - Parameters: - /// - textView: The text view to use. - /// - layer: The language layer to perform the query on. - /// - layerSet: The set of layers that exist in the document. - /// Used for efficient lookup of existing `(language, range)` pairs - /// - touchedLayers: The set of layers that existed before updating injected layers. - /// Will have items removed as they are found. - /// - readBlock: A completion block for reading from text storage efficiently. - /// - Returns: An index set of any updated indexes. - @discardableResult - internal func updateInjectedLanguageLayers( - textView: HighlighterTextView, - layer: LanguageLayer, - layerSet: inout Set, - touchedLayers: inout Set, - readBlock: @escaping Parser.ReadBlock - ) -> IndexSet { - guard let tree = layer.tree, - let rootNode = tree.rootNode, - let cursor = layer.languageQuery?.execute(node: rootNode, in: tree) else { - return IndexSet() + /// - editState: The edit state to apply. + /// - startAtLayerIndex: An optional layer index to start from if some work has already been done on this edit + /// state object. + /// - runningAsync: Determine whether or not to timeout long running parse tasks. + internal func applyEdit(editState: EditState, startAtLayerIndex: Int? = nil, runningAsync: Bool = false) { + stateLock.lock() + defer { + stateLock.unlock() } + + print("Applying Edit", runningAsync ? "(Async)" : "") + guard let readBlock, let textView else { return } + + // Loop through all layers, apply edits & find changed byte ranges. + let startIdx = startAtLayerIndex ?? 0 + for layerIdx in (startIdx.. Void) + ) { + stateLock.lock() + defer { + stateLock.unlock() + } + guard let textView else { return } + + var highlights: [HighlightRange] = [] + var injectedSet = IndexSet(integersIn: range) + + for layer in state.layers where layer.id != state.primaryLayer { + // Query injected only if a layer's ranges intersects with `range` + for layerRange in layer.ranges { + if let rangeIntersection = range.intersection(layerRange) { + highlights.append(contentsOf: queryLayerHighlights( + layer: layer, + textView: textView, + range: rangeIntersection + )) + + injectedSet.remove(integersIn: rangeIntersection) + } + } + } + + // Query primary for any ranges that weren't used in the injected layers. + for range in injectedSet.rangeView { + highlights.append(contentsOf: queryLayerHighlights( + layer: state.layers[0], + textView: textView, + range: NSRange(range) + )) + } + + if !runningAsync { + completion(highlights) + } else { + DispatchQueue.main.async { + completion(highlights) + } + } + } + + internal func queryHighlightsForRangeAsync( + range: NSRange, + completion: @escaping (([HighlightRange]) -> Void) + ) { + let id = UUID() + print("\tQueueing Query Async \(id). Items in queue: \(queuedEdits.count + queuedQueries.count)") + queuedQueries.append { [weak self] in + print("\tAsync Query Dequeued \(id)", range, self == nil ? "No Self!" : "") + self?.queryHighlightsForRange(range: range, runningAsync: true, completion: completion) + } + beginTasksIfNeeded() + } + /// Queries the given language layer for any highlights. /// - Parameters: /// - layer: The layer to query. @@ -18,21 +78,12 @@ extension TreeSitterClient { /// - range: The range to query for. /// - Returns: Any ranges to highlight. internal func queryLayerHighlights( - layer: LanguageLayer, + layer: TSLanguageLayer, textView: HighlighterTextView, range: NSRange ) -> [HighlightRange] { - // Make sure we don't change the tree while we copy it. - self.semaphore.wait() - - guard let tree = layer.tree?.copy() else { - self.semaphore.signal() - return [] - } - - self.semaphore.signal() - - guard let rootNode = tree.rootNode else { + guard let tree = layer.tree, + let rootNode = tree.rootNode else { return [] } @@ -61,37 +112,11 @@ extension TreeSitterClient { // Sometimes `cursor.setRange` just doesnt work :( so we have to do a redundant check for a valid range // in the included range let intersectionRange = $0.range.intersection(includedRange) ?? .zero - // Some languages add an "@spell" capture to indicate a portion of text that should be spellchecked - // (usually comments). But this causes other captures in the same range to be overriden. So we ignore - // that specific capture type. - if intersectionRange.length > 0 && $0.name != "spell" && $0.name != "injection.content" { - return HighlightRange(range: intersectionRange, capture: CaptureName.fromString($0.name ?? "")) + // Check that the capture name is one CETV can parse. If not, ignore it completely. + if intersectionRange.length > 0, let captureName = CaptureName.fromString($0.name ?? "") { + return HighlightRange(range: intersectionRange, capture: captureName) } return nil } } - - /// Returns all injected languages from a given cursor. The cursor must be new, - /// having not been used for normal highlight matching. - /// - Parameters: - /// - cursor: The cursor to use for finding injected languages. - /// - textProvider: A callback for efficiently fetching text. - /// - Returns: A map of each language to all the ranges they have been injected into. - internal func injectedLanguagesFrom( - cursor: QueryCursor, - textProvider: @escaping ResolvingQueryCursor.TextProvider - ) -> [String: [NamedRange]] { - var languages: [String: [NamedRange]] = [:] - - for match in cursor { - if let injection = match.injection(with: textProvider) { - if languages[injection.name] == nil { - languages[injection.name] = [] - } - languages[injection.name]?.append(injection) - } - } - - return languages - } } diff --git a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+LanguageLayer.swift b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+LanguageLayer.swift deleted file mode 100644 index c5ab2cc15..000000000 --- a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+LanguageLayer.swift +++ /dev/null @@ -1,54 +0,0 @@ -// -// TreeSitterClient+LanguageLayer.swift -// -// -// Created by Khan Winter on 3/8/23. -// - -import Foundation -import CodeEditLanguages -import SwiftTreeSitter - -extension TreeSitterClient { - class LanguageLayer: Hashable { - /// Initialize a language layer - /// - Parameters: - /// - id: The ID of the layer. - /// - parser: A parser to use for the layer. - /// - supportsInjections: Set to true when the langauge supports the `injections` query. - /// - tree: The tree-sitter tree generated while editing/parsing a document. - /// - languageQuery: The language query used for fetching the associated `queries.scm` file - /// - ranges: All ranges this layer acts on. Must be kept in order and w/o overlap. - init( - id: TreeSitterLanguage, - parser: Parser, - supportsInjections: Bool, - tree: Tree? = nil, - languageQuery: Query? = nil, - ranges: [NSRange] - ) { - self.id = id - self.parser = parser - self.supportsInjections = supportsInjections - self.tree = tree - self.languageQuery = languageQuery - self.ranges = ranges - } - - let id: TreeSitterLanguage - let parser: Parser - let supportsInjections: Bool - var tree: Tree? - var languageQuery: Query? - var ranges: [NSRange] - - static func == (lhs: TreeSitterClient.LanguageLayer, rhs: TreeSitterClient.LanguageLayer) -> Bool { - return lhs.id == rhs.id && lhs.ranges == rhs.ranges - } - - func hash(into hasher: inout Hasher) { - hasher.combine(id) - hasher.combine(ranges) - } - } -} diff --git a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift index 66276576f..456bc5547 100644 --- a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift +++ b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift @@ -18,19 +18,25 @@ import SwiftTreeSitter /// However, the `setText` method will re-compile the entire corpus so should be used sparingly. final class TreeSitterClient: HighlightProviding { + typealias AsyncCallback = @Sendable () -> Void + // MARK: - Properties/Constants public var identifier: String { "CodeEdit.TreeSitterClient" } - internal var primaryLayer: TreeSitterLanguage - internal var layers: [LanguageLayer] = [] + internal weak var textView: HighlighterTextView? + internal var readBlock: Parser.ReadBlock? - internal var textProvider: ResolvingQueryCursor.TextProvider + internal var runningTask: Task? + internal var stateLock: NSLock = NSLock() + internal var queueLock: NSLock = NSLock() + internal var queuedEdits: [AsyncCallback] = [] + internal var queuedQueries: [AsyncCallback] = [] - /// Used to ensure safe use of the shared tree-sitter tree state in different sync/async contexts. - internal let semaphore: DispatchSemaphore = DispatchSemaphore(value: 1) + internal var state: TreeSitterState + internal var textProvider: ResolvingQueryCursor.TextProvider internal enum Constants { /// The maximum amount of limits a cursor can match during a query. @@ -40,6 +46,18 @@ final class TreeSitterClient: HighlightProviding { /// See: https://github.com/neovim/neovim/issues/14897 /// And: https://github.com/helix-editor/helix/pull/4830 static let treeSitterMatchLimit = 256 + + /// The timeout for parsers. + static let parserTimeout: TimeInterval = 0.005 + + /// The maximum length of an edit before it must be processed asynchronously + static let maxSyncEditLength: Int = 1024 + + /// The maximum length a document can be before all queries and edits must be processed asynchronously. + static let maxSyncContentLength: Int = 1_000_000 + + /// The maximum length a query can be before it must be performed asynchronously. + static let maxSyncQueryLength: Int = 4096 } // MARK: - Init/Config @@ -50,59 +68,27 @@ final class TreeSitterClient: HighlightProviding { /// - textProvider: The text provider callback to read any text. public init(codeLanguage: CodeLanguage, textProvider: @escaping ResolvingQueryCursor.TextProvider) { self.textProvider = textProvider - self.primaryLayer = codeLanguage.id - setLanguage(codeLanguage: codeLanguage) + self.state = TreeSitterState(primaryLayer: codeLanguage.id) + self.setLanguage(codeLanguage: codeLanguage) } /// Sets the primary language for the client. Will reset all layers, will not do any parsing work. /// - Parameter codeLanguage: The new primary language. public func setLanguage(codeLanguage: CodeLanguage) { - // Remove all trees and languages, everything needs to be re-parsed. - layers.removeAll() - - primaryLayer = codeLanguage.id - layers = [ - LanguageLayer( - id: codeLanguage.id, - parser: Parser(), - supportsInjections: codeLanguage.additionalHighlights?.contains("injections") ?? false, - tree: nil, - languageQuery: TreeSitterModel.shared.query(for: codeLanguage.id), - ranges: [] - ) - ] - - if let treeSitterLanguage = codeLanguage.language { - try? layers[0].parser.setLanguage(treeSitterLanguage) - } + cancelAllRunningTasks() + state.setLanguage(codeLanguage: codeLanguage) } // MARK: - HighlightProviding /// Set up and parse the initial language tree and all injected layers. func setUp(textView: HighlighterTextView) { - let readBlock = createReadBlock(textView: textView) - - layers[0].tree = createTree( - parser: layers[0].parser, - readBlock: readBlock - ) - - var layerSet = Set(arrayLiteral: layers[0]) - var touchedLayers = Set() - - var idx = 0 - while idx < layers.count { - updateInjectedLanguageLayers( - textView: textView, - layer: layers[idx], - layerSet: &layerSet, - touchedLayers: &touchedLayers, - readBlock: readBlock - ) - - idx += 1 - } + cancelAllRunningTasks() + + self.textView = textView + readBlock = textView.createReadBlock() + + state.setUp(textView: textView) } /// Notifies the highlighter of an edit and in exchange gets a set of indices that need to be re-highlighted. @@ -118,75 +104,22 @@ final class TreeSitterClient: HighlightProviding { delta: Int, completion: @escaping ((IndexSet) -> Void) ) { + print("Received Edit", range, delta) guard let edit = InputEdit(range: range, delta: delta, oldEndPoint: .zero) else { return } - let readBlock = createReadBlock(textView: textView) - var rangeSet = IndexSet() - - // Helper data structure for finding existing layers in O(1) when adding injected layers - var layerSet = Set(minimumCapacity: layers.count) - // Tracks which layers were not touched at some point during the edit. Any layers left in this set - // after the second loop are removed. - var touchedLayers = Set(minimumCapacity: layers.count) - - // Loop through all layers, apply edits & find changed byte ranges. - for layerIdx in (0.. Constants.maxSyncEditLength + let longDocument = textView.documentRange.length > Constants.maxSyncContentLength + + if hasOutstandingWork || longEdit || longDocument { + print("\tQueueing Async") + applyEditAsync(editState: EditState(edit: edit, completion: completion), startAtLayerIndex: 0) + queueLock.unlock() + } else { + queueLock.unlock() + print("Performing Sync") + applyEdit(editState: EditState(edit: edit, completion: completion)) } - - // Loop again and apply injections query, add any ranges not previously found - // using while loop because `updateInjectedLanguageLayers` can add to `layers` during the loop - var idx = 0 - while idx < layers.count { - let layer = layers[idx] - - if layer.supportsInjections { - rangeSet.formUnion( - updateInjectedLanguageLayers( - textView: textView, - layer: layer, - layerSet: &layerSet, - touchedLayers: &touchedLayers, - readBlock: readBlock - ) - ) - } - - idx += 1 - } - - // Delete any layers that weren't touched at some point during the edit. - layers.removeAll(where: { touchedLayers.contains($0) }) - - completion(rangeSet) } /// Initiates a highlight query. @@ -197,94 +130,81 @@ final class TreeSitterClient: HighlightProviding { func queryHighlightsFor( textView: HighlighterTextView, range: NSRange, - completion: @escaping ((([HighlightRange]) -> Void)) + completion: @escaping (([HighlightRange]) -> Void) ) { - var highlights: [HighlightRange] = [] - var injectedSet = IndexSet(integersIn: range) - - for layer in layers where layer.id != primaryLayer { - // Query injected only if a layer's ranges intersects with `range` - for layerRange in layer.ranges { - if let rangeIntersection = range.intersection(layerRange) { - highlights.append(contentsOf: queryLayerHighlights( - layer: layer, - textView: textView, - range: rangeIntersection - )) - - injectedSet.remove(integersIn: rangeIntersection) - } - } + print("Received Query", range) + queueLock.lock() + let longQuery = range.length > Constants.maxSyncQueryLength + let longDocument = textView.documentRange.length > Constants.maxSyncContentLength + + if hasOutstandingWork || longQuery || longDocument { + print("\tQueuing Async Query") + queryHighlightsForRangeAsync(range: range, completion: completion) + queueLock.unlock() + } else { + queueLock.unlock() + print("\tQueuing Sync Query") + queryHighlightsForRange(range: range, runningAsync: false, completion: completion) } + } - // Query primary for any ranges that weren't used in the injected layers. - for range in injectedSet.rangeView { - highlights.append(contentsOf: queryLayerHighlights( - layer: layers[0], - textView: textView, - range: NSRange(range) - )) - } + // MARK: - Async Helpers - completion(highlights) + var hasOutstandingWork: Bool { + runningTask != nil || queuedEdits.count > 0 || queuedQueries.count > 0 } - // MARK: - Helpers + /// Spawn the running task if one is needed and doesn't already exist. + internal func beginTasksIfNeeded() { + guard runningTask == nil && (queuedEdits.count > 0 || queuedQueries.count > 0) else { return } + runningTask = Task.detached(priority: .userInitiated) { + defer { + self.runningTask = nil + } - /// Attempts to create a language layer and load a highlights file. - /// Adds the layer to the `layers` array if successful. - /// - Parameters: - /// - layerId: A language ID to add as a layer. - /// - readBlock: Completion called for efficient string lookup. - internal func addLanguageLayer( - layerId: TreeSitterLanguage, - readBlock: @escaping Parser.ReadBlock - ) -> LanguageLayer? { - guard let language = CodeLanguage.allLanguages.first(where: { $0.id == layerId }), - let parserLanguage = language.language - else { - return nil - } + var info = mach_timebase_info() + guard mach_timebase_info(&info) == KERN_SUCCESS else { return } - let newLayer = LanguageLayer( - id: layerId, - parser: Parser(), - supportsInjections: language.additionalHighlights?.contains("injections") ?? false, - tree: nil, - languageQuery: TreeSitterModel.shared.query(for: layerId), - ranges: [] - ) - - do { - try newLayer.parser.setLanguage(parserLanguage) - } catch { - return nil - } + let start = mach_absolute_time() - layers.append(newLayer) - return newLayer - } + do { + while let nextQueuedTask = self.determineNextTask() { + let end = mach_absolute_time() - /// Creates a tree-sitter tree. - /// - Parameters: - /// - parser: The parser object to use to parse text. - /// - readBlock: A callback for fetching blocks of text. - /// - Returns: A tree if it could be parsed. - internal func createTree(parser: Parser, readBlock: @escaping Parser.ReadBlock) -> Tree? { - return parser.parse(tree: nil, readBlock: readBlock) + let elapsed = end - start + + let nanos = elapsed * UInt64(info.numer) / UInt64(info.denom) + + print("\t\tFound Next Task. Tasks Remaining:", self.queuedEdits.count + self.queuedQueries.count) + print("\t\tTime Since queue started (ms): ", TimeInterval(nanos) / TimeInterval(NSEC_PER_MSEC)) + try Task.checkCancellation() + nextQueuedTask() + } + } catch { } + } } - internal func createReadBlock(textView: HighlighterTextView) -> Parser.ReadBlock { - return { byteOffset, _ in - let limit = textView.documentRange.length - let location = byteOffset / 2 - let end = min(location + (1024), limit) - if location > end { - // Ignore and return nothing, tree-sitter's internal tree can be incorrect in some situations. - return nil - } - let range = NSRange(location.. AsyncCallback? { + queueLock.lock() + defer { + queueLock.unlock() } + + // Get an edit task if any, otherwise get a highlight task if any. + if queuedEdits.count > 0 { + return queuedEdits.removeFirst() + } else if queuedQueries.count > 0 { + return queuedQueries.removeFirst() + } else { + return nil + } + } + + /// Cancels all running and enqueued tasks. + private func cancelAllRunningTasks() { + runningTask?.cancel() + queuedEdits.removeAll() + queuedQueries.removeAll() } } diff --git a/Sources/CodeEditTextView/TreeSitter/TreeSitterState.swift b/Sources/CodeEditTextView/TreeSitter/TreeSitterState.swift new file mode 100644 index 000000000..4f5a35552 --- /dev/null +++ b/Sources/CodeEditTextView/TreeSitter/TreeSitterState.swift @@ -0,0 +1,255 @@ +// +// TreeSitterState.swift +// CodeEditTextView +// +// Created by Khan Winter on 5/21/23. +// + +import Foundation +import SwiftTreeSitter +import CodeEditLanguages + +/// TreeSitterState contains the tree of language layers that make up the tree-sitter document. +/// Use the given +class TreeSitterState { + private(set) var primaryLayer: TreeSitterLanguage + private(set) var layers: [TSLanguageLayer] = [] + + // MARK: - Init + + init(primaryLayer: TreeSitterLanguage) { + self.primaryLayer = primaryLayer + } + + /// Initialize the state object with the given read block. + /// - Parameter textView: The textView to use as a data source. + public func setUp(textView: HighlighterTextView) { + let readBlock = textView.createReadBlock() + + let previousTimeout = layers[0].parser.timeout + layers[0].parser.timeout = 0.0 + layers[0].tree = layers[0].parser.createTree(readBlock: readBlock) + layers[0].parser.timeout = previousTimeout + + var layerSet = Set(arrayLiteral: layers[0]) + var touchedLayers = Set() + + var idx = 0 + while idx < layers.count { + updateInjectedLanguageLayer( + textView: textView, + layer: layers[idx], + layerSet: &layerSet, + touchedLayers: &touchedLayers + ) + + idx += 1 + } + } + + /// Sets the primary language for the client. Will reset all layers, will not do any parsing work. + /// - Parameter codeLanguage: The new primary language. + public func setLanguage(codeLanguage: CodeLanguage) { + // Remove all trees and languages, everything needs to be re-parsed. + layers.removeAll() + + primaryLayer = codeLanguage.id + layers = [ + TSLanguageLayer( + id: codeLanguage.id, + parser: Parser(), + supportsInjections: codeLanguage.additionalHighlights?.contains("injections") ?? false, + tree: nil, + languageQuery: TreeSitterModel.shared.query(for: codeLanguage.id), + ranges: [] + ) + ] + + if let treeSitterLanguage = codeLanguage.language { + try? layers[0].parser.setLanguage(treeSitterLanguage) + } + } + + // MARK: - Layer Management + + /// Removes a layer at the given index. + /// - Parameter idx: The index of the layer to remove. + public func removeLanguageLayer(at idx: Int) { + layers.remove(at: idx) + } + + public func removeLanguageLayers(in set: Set) { + layers.removeAll(where: { set.contains($0 )}) + } + + /// Attempts to create a language layer and load a highlights file. + /// Adds the layer to the `layers` array if successful. + /// - Parameters: + /// - layerId: A language ID to add as a layer. + /// - readBlock: Completion called for efficient string lookup. + public func addLanguageLayer( + layerId: TreeSitterLanguage, + readBlock: @escaping Parser.ReadBlock + ) -> TSLanguageLayer? { + guard let language = CodeLanguage.allLanguages.first(where: { $0.id == layerId }), + let parserLanguage = language.language + else { + return nil + } + + let newLayer = TSLanguageLayer( + id: layerId, + parser: Parser(), + supportsInjections: language.additionalHighlights?.contains("injections") ?? false, + tree: nil, + languageQuery: TreeSitterModel.shared.query(for: layerId), + ranges: [] + ) + + do { + try newLayer.parser.setLanguage(parserLanguage) + } catch { + return nil + } + + layers.append(newLayer) + return newLayer + } + + // MARK: - Injection Layers + + /// Inserts any new language layers, and removes any that may have been deleted after an edit. + /// - Parameters: + /// - textView: The data source for text ranges. + /// - touchedLayers: A set of layers. Each time a layer is visited, it will be removed from this set. + /// Use this to determine if any layers were not modified after this method was run. + /// Those layers should be removed. + /// - Returns: A set of indices of any new layers. This set indicates ranges that should be re-highlighted. + public func updateInjectedLayers( + textView: HighlighterTextView, + touchedLayers: Set + ) -> IndexSet { + var layerSet = Set(layers) + var touchedLayers = touchedLayers + var rangeSet = IndexSet() + + // Loop through each layer and apply injections query, add any ranges not previously found + // using while loop because `updateInjectedLanguageLayer` can add to `layers` during the loop + var idx = 0 + while idx < layers.count { + let layer = layers[idx] + + if layer.supportsInjections { + rangeSet.formUnion( + updateInjectedLanguageLayer( + textView: textView, + layer: layer, + layerSet: &layerSet, + touchedLayers: &touchedLayers + ) + ) + } + + idx += 1 + } + + // Delete any layers that weren't touched at some point during the edit. + removeLanguageLayers(in: touchedLayers) + + return rangeSet + } + + /// Performs an injections query on the given language layer. + /// Updates any existing layers with new ranges and adds new layers if needed. + /// - Parameters: + /// - textView: The text view to use. + /// - layer: The language layer to perform the query on. + /// - layerSet: The set of layers that exist in the document. + /// Used for efficient lookup of existing `(language, range)` pairs + /// - touchedLayers: The set of layers that existed before updating injected layers. + /// Will have items removed as they are found. + /// - Returns: An index set of any updated indexes. + @discardableResult + private func updateInjectedLanguageLayer( + textView: HighlighterTextView, + layer: TSLanguageLayer, + layerSet: inout Set, + touchedLayers: inout Set + ) -> IndexSet { + guard let tree = layer.tree, + let rootNode = tree.rootNode, + let cursor = layer.languageQuery?.execute(node: rootNode, in: tree) else { + return IndexSet() + } + + cursor.matchLimit = TreeSitterClient.Constants.treeSitterMatchLimit + + let languageRanges = self.injectedLanguagesFrom(cursor: cursor) { range, _ in + return textView.stringForRange(range) + } + + var updatedRanges = IndexSet() + + for (languageName, ranges) in languageRanges { + guard let treeSitterLanguage = TreeSitterLanguage(rawValue: languageName) else { + continue + } + + if treeSitterLanguage == primaryLayer { + continue + } + + for range in ranges { + // Temp layer object + let layer = TSLanguageLayer( + id: treeSitterLanguage, + parser: Parser(), + supportsInjections: false, + ranges: [range.range] + ) + + if layerSet.contains(layer) { + // If we've found this layer, it means it should exist after an edit. + touchedLayers.remove(layer) + } else { + let readBlock = textView.createReadBlock() + // New range, make a new layer! + if let addedLayer = addLanguageLayer(layerId: treeSitterLanguage, readBlock: readBlock) { + addedLayer.ranges = [range.range] + addedLayer.parser.includedRanges = addedLayer.ranges.map { $0.tsRange } + addedLayer.tree = addedLayer.parser.createTree(readBlock: readBlock) + + layerSet.insert(addedLayer) + updatedRanges.insert(range: range.range) + } + } + } + } + + return updatedRanges + } + + /// Returns all injected languages from a given cursor. The cursor must be new, + /// having not been used for normal highlight matching. + /// - Parameters: + /// - cursor: The cursor to use for finding injected languages. + /// - textProvider: A callback for efficiently fetching text. + /// - Returns: A map of each language to all the ranges they have been injected into. + private func injectedLanguagesFrom( + cursor: QueryCursor, + textProvider: @escaping ResolvingQueryCursor.TextProvider + ) -> [String: [NamedRange]] { + var languages: [String: [NamedRange]] = [:] + + for match in cursor { + if let injection = match.injection(with: textProvider) { + if languages[injection.name] == nil { + languages[injection.name] = [] + } + languages[injection.name]?.append(injection) + } + } + + return languages + } +} From 4a3ed1944c05a0ad7ef8cd77e249c36c232b882f Mon Sep 17 00:00:00 2001 From: Khan Winter <35942988+thecoolwinter@users.noreply.github.com> Date: Mon, 22 May 2023 22:49:40 -0500 Subject: [PATCH 02/10] Fix deadlocks, remove debugs Removes a lot of deadlocks from the initial implementation using locks. Also imports the `os` system library to use the newer `OSAllocatedUnfairLock` type for speed. --- Package.swift | 2 +- .../STTextViewController+Highlighter.swift | 2 +- .../STTextViewController+TextFormation.swift | 4 +- .../Highlighting/HighlightProviding.swift | 10 +-- .../Highlighting/Highlighter.swift | 40 +++------- .../TreeSitter/TreeSitterClient+Edit.swift | 21 ++--- .../TreeSitterClient+Highlight.swift | 14 +--- .../TreeSitter/TreeSitterClient.swift | 76 +++++++++---------- .../TreeSitter/TreeSitterState.swift | 21 ++--- 9 files changed, 71 insertions(+), 119 deletions(-) diff --git a/Package.swift b/Package.swift index 244bfd4ad..de8c2443f 100644 --- a/Package.swift +++ b/Package.swift @@ -19,7 +19,7 @@ let package = Package( ), .package( url: "https://github.com/CodeEditApp/CodeEditLanguages.git", - exact: "0.1.13" + exact: "0.1.14" ), .package( url: "https://github.com/lukepistrol/SwiftLintPlugin", diff --git a/Sources/CodeEditTextView/Controller/STTextViewController+Highlighter.swift b/Sources/CodeEditTextView/Controller/STTextViewController+Highlighter.swift index 75bf0b6ae..e1d113597 100644 --- a/Sources/CodeEditTextView/Controller/STTextViewController+Highlighter.swift +++ b/Sources/CodeEditTextView/Controller/STTextViewController+Highlighter.swift @@ -31,7 +31,7 @@ extension STTextViewController { return self?.textView.textContentStorage?.textStorage?.mutableString.substring(with: range) } - provider = TreeSitterClient(codeLanguage: language, textProvider: textProvider) + provider = TreeSitterClient(textProvider: textProvider) } if let provider = provider { diff --git a/Sources/CodeEditTextView/Filters/STTextViewController+TextFormation.swift b/Sources/CodeEditTextView/Filters/STTextViewController+TextFormation.swift index 5e24773ed..67fb03e52 100644 --- a/Sources/CodeEditTextView/Filters/STTextViewController+TextFormation.swift +++ b/Sources/CodeEditTextView/Filters/STTextViewController+TextFormation.swift @@ -107,11 +107,11 @@ extension STTextViewController { range: range, limit: textView.textContentStorage?.length ?? 0) - textView.undoManager?.beginUndoGrouping() +// textView.undoManager?.beginUndoGrouping() let result = shouldApplyMutation(mutation, to: textView) - textView.undoManager?.endUndoGrouping() +// textView.undoManager?.endUndoGrouping() return result } diff --git a/Sources/CodeEditTextView/Highlighting/HighlightProviding.swift b/Sources/CodeEditTextView/Highlighting/HighlightProviding.swift index 199c2cc42..88ed36ea8 100644 --- a/Sources/CodeEditTextView/Highlighting/HighlightProviding.swift +++ b/Sources/CodeEditTextView/Highlighting/HighlightProviding.swift @@ -11,19 +11,17 @@ import STTextView import AppKit /// The protocol a class must conform to to be used for highlighting. -public protocol HighlightProviding { +public protocol HighlightProviding: AnyObject { /// A unique identifier for the highlighter object. /// Example: `"CodeEdit.TreeSitterHighlighter"` /// - Note: This does not need to be *globally* unique, merely unique across all the highlighters used. var identifier: String { get } - /// Called once at editor initialization. - func setUp(textView: HighlighterTextView) - - /// Updates the highlighter's code language. + /// Called once to set up the highlight provider with a data source and language. /// - Parameters: + /// - textView: The text view to use as a text source. /// - codeLanguage: The langugage that should be used by the highlighter. - func setLanguage(codeLanguage: CodeLanguage) + func setUp(textView: HighlighterTextView, codeLanguage: CodeLanguage) /// Notifies the highlighter of an edit and in exchange gets a set of indices that need to be re-highlighted. /// The returned `IndexSet` should include all indexes that need to be highlighted, including any inserted text. diff --git a/Sources/CodeEditTextView/Highlighting/Highlighter.swift b/Sources/CodeEditTextView/Highlighting/Highlighter.swift index 157fc353a..a659fed7b 100644 --- a/Sources/CodeEditTextView/Highlighting/Highlighter.swift +++ b/Sources/CodeEditTextView/Highlighting/Highlighter.swift @@ -67,11 +67,13 @@ class Highlighter: NSObject { /// - textView: The text view to highlight. /// - treeSitterClient: The tree-sitter client to handle tree updates and highlight queries. /// - theme: The theme to use for highlights. - init(textView: STTextView, - highlightProvider: HighlightProviding?, - theme: EditorTheme, - attributeProvider: ThemeAttributesProviding, - language: CodeLanguage) { + init( + textView: STTextView, + highlightProvider: HighlightProviding?, + theme: EditorTheme, + attributeProvider: ThemeAttributesProviding, + language: CodeLanguage + ) { self.textView = textView self.highlightProvider = highlightProvider self.theme = theme @@ -80,15 +82,13 @@ class Highlighter: NSObject { super.init() - highlightProvider?.setLanguage(codeLanguage: language) - guard textView.textContentStorage?.textStorage != nil else { assertionFailure("Text view does not have a textStorage") return } textView.textContentStorage?.textStorage?.delegate = self - highlightProvider?.setUp(textView: textView) + highlightProvider?.setUp(textView: textView, codeLanguage: language) if let scrollView = textView.enclosingScrollView { NotificationCenter.default.addObserver(self, @@ -113,7 +113,7 @@ class Highlighter: NSObject { /// Sets the language and causes a re-highlight of the entire text. /// - Parameter language: The language to update to. public func setLanguage(language: CodeLanguage) { - highlightProvider?.setLanguage(codeLanguage: language) + highlightProvider?.setUp(textView: textView, codeLanguage: language) invalidate() } @@ -121,8 +121,7 @@ class Highlighter: NSObject { /// - Parameter provider: The provider to use for future syntax highlights. public func setHighlightProvider(_ provider: HighlightProviding) { self.highlightProvider = provider - highlightProvider?.setLanguage(codeLanguage: language) - highlightProvider?.setUp(textView: textView) + highlightProvider?.setUp(textView: textView, codeLanguage: language) invalidate() } @@ -167,10 +166,8 @@ private extension Highlighter { func highlight(range rangeToHighlight: NSRange) { pendingSet.insert(integersIn: rangeToHighlight) - print("--Querying Range", rangeToHighlight, visibleSet.rangeView.map { $0 }) highlightProvider?.queryHighlightsFor(textView: self.textView, range: rangeToHighlight) { [weak self] highlightRanges in - print("--Received Query", rangeToHighlight, self!.visibleSet.rangeView.map { $0 }) guard let attributeProvider = self?.attributeProvider, let textView = self?.textView else { return } @@ -282,21 +279,10 @@ extension Highlighter: NSTextStorageDelegate { if delta > 0 { visibleSet.insert(range: editedRange) } - var info = mach_timebase_info() - guard mach_timebase_info(&info) == KERN_SUCCESS else { return } - - let start = mach_absolute_time() highlightProvider?.applyEdit(textView: self.textView, range: range, delta: delta) { [weak self] invalidatedIndexSet in - let end = mach_absolute_time() - - let elapsed = end - start - - let nanos = elapsed * UInt64(info.numer) / UInt64(info.denom) - print("Receiving Results", TimeInterval(nanos) / TimeInterval(NSEC_PER_MSEC)) - let indexSet = invalidatedIndexSet .union(IndexSet(integersIn: editedRange)) // Only invalidate indices that are visible. @@ -306,11 +292,5 @@ extension Highlighter: NSTextStorageDelegate { self?.invalidate(range: NSRange(range)) } } - let end = mach_absolute_time() - - let elapsed = end - start - - let nanos = elapsed * UInt64(info.numer) / UInt64(info.denom) - print("Returning From Function", TimeInterval(nanos) / TimeInterval(NSEC_PER_MSEC)) } } diff --git a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+Edit.swift b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+Edit.swift index 10dc8bc6b..7f9238840 100644 --- a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+Edit.swift +++ b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+Edit.swift @@ -10,7 +10,7 @@ import SwiftTreeSitter import CodeEditLanguages extension TreeSitterClient { - /// A helper class for passing edit state from synchronous to asynchronous contexts. + /// This class contains an edit state that can be resumed if a parser hits a timeout. class EditState { var edit: InputEdit var rangeSet: IndexSet @@ -38,13 +38,8 @@ extension TreeSitterClient { /// state object. /// - runningAsync: Determine whether or not to timeout long running parse tasks. internal func applyEdit(editState: EditState, startAtLayerIndex: Int? = nil, runningAsync: Bool = false) { + guard let readBlock, let textView, let state else { return } stateLock.lock() - defer { - stateLock.unlock() - } - - print("Applying Edit", runningAsync ? "(Async)" : "") - guard let readBlock, let textView else { return } // Loop through all layers, apply edits & find changed byte ranges. let startIdx = startAtLayerIndex ?? 0 @@ -80,12 +75,10 @@ extension TreeSitterClient { ) } catch { // Timed out, queue an async edit with any data already computed. - print("\tCaught Timeout Error", layer.parser.timeout) if !runningAsync { applyEditAsync(editState: editState, startAtLayerIndex: layerIdx) - } else { - assertionFailure("`layer.findChangedByteRanges` should never throw when `timeout = nil`.") } + stateLock.unlock() return } @@ -97,6 +90,7 @@ extension TreeSitterClient { state.updateInjectedLayers(textView: textView, touchedLayers: editState.touchedLayers) ) + stateLock.unlock() if runningAsync { DispatchQueue.main.async { editState.completion(editState.rangeSet) @@ -109,11 +103,8 @@ extension TreeSitterClient { /// Enqueues the given edit state to be applied asynchronously. /// - Parameter editState: The edit state to enqueue. internal func applyEditAsync(editState: EditState, startAtLayerIndex: Int) { - let id = UUID() - print("\tQueueing Async \(id). Items in queue: \(queuedEdits.count + queuedQueries.count)") - queuedEdits.append { [weak self] in - print("\tAsync Edit Dequeued \(id)") - self?.applyEdit(editState: editState, startAtLayerIndex: startAtLayerIndex, runningAsync: true) + queuedEdits.append { + self.applyEdit(editState: editState, startAtLayerIndex: startAtLayerIndex, runningAsync: true) } beginTasksIfNeeded() } diff --git a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+Highlight.swift b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+Highlight.swift index c3e1aeb11..7a351451a 100644 --- a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+Highlight.swift +++ b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+Highlight.swift @@ -10,17 +10,13 @@ import SwiftTreeSitter import CodeEditLanguages extension TreeSitterClient { - internal func queryHighlightsForRange( range: NSRange, runningAsync: Bool, completion: @escaping (([HighlightRange]) -> Void) ) { + guard let textView, let state else { return } stateLock.lock() - defer { - stateLock.unlock() - } - guard let textView else { return } var highlights: [HighlightRange] = [] var injectedSet = IndexSet(integersIn: range) @@ -49,6 +45,7 @@ extension TreeSitterClient { )) } + stateLock.unlock() if !runningAsync { completion(highlights) } else { @@ -62,11 +59,8 @@ extension TreeSitterClient { range: NSRange, completion: @escaping (([HighlightRange]) -> Void) ) { - let id = UUID() - print("\tQueueing Query Async \(id). Items in queue: \(queuedEdits.count + queuedQueries.count)") - queuedQueries.append { [weak self] in - print("\tAsync Query Dequeued \(id)", range, self == nil ? "No Self!" : "") - self?.queryHighlightsForRange(range: range, runningAsync: true, completion: completion) + queuedQueries.append { + self.queryHighlightsForRange(range: range, runningAsync: true, completion: completion) } beginTasksIfNeeded() } diff --git a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift index 456bc5547..d9cf87c0a 100644 --- a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift +++ b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift @@ -6,6 +6,7 @@ // import Foundation +import os import CodeEditLanguages import SwiftTreeSitter @@ -17,7 +18,6 @@ import SwiftTreeSitter /// `textProvider` callback. You can optionally update the text manually using the `setText` method. /// However, the `setText` method will re-compile the entire corpus so should be used sparingly. final class TreeSitterClient: HighlightProviding { - typealias AsyncCallback = @Sendable () -> Void // MARK: - Properties/Constants @@ -26,16 +26,25 @@ final class TreeSitterClient: HighlightProviding { "CodeEdit.TreeSitterClient" } + /// The text view to use as a data source for text. internal weak var textView: HighlighterTextView? + /// A callback to use to efficiently fetch portions of text. internal var readBlock: Parser.ReadBlock? + /// The running background task. internal var runningTask: Task? - internal var stateLock: NSLock = NSLock() - internal var queueLock: NSLock = NSLock() + /// An array of all edits queued for execution. internal var queuedEdits: [AsyncCallback] = [] + /// An array of all highlight queries queued for execution. internal var queuedQueries: [AsyncCallback] = [] - internal var state: TreeSitterState + /// A lock that must be obtained whenever `state` is modified + internal var stateLock: OSAllocatedUnfairLock = OSAllocatedUnfairLock() + /// A lock that must be obtained whenever either `queuedEdits` or `queuedHighlights` is modified + internal var queueLock: OSAllocatedUnfairLock = OSAllocatedUnfairLock() + + /// The internal tree-sitter layer tree object. + internal var state: TreeSitterState? internal var textProvider: ResolvingQueryCursor.TextProvider internal enum Constants { @@ -64,31 +73,36 @@ final class TreeSitterClient: HighlightProviding { /// Initializes the `TreeSitterClient` with the given parameters. /// - Parameters: + /// - textView: The text view to use as a data source. /// - codeLanguage: The language to set up the parser with. /// - textProvider: The text provider callback to read any text. - public init(codeLanguage: CodeLanguage, textProvider: @escaping ResolvingQueryCursor.TextProvider) { + public init(textProvider: @escaping ResolvingQueryCursor.TextProvider) { self.textProvider = textProvider - self.state = TreeSitterState(primaryLayer: codeLanguage.id) - self.setLanguage(codeLanguage: codeLanguage) - } - - /// Sets the primary language for the client. Will reset all layers, will not do any parsing work. - /// - Parameter codeLanguage: The new primary language. - public func setLanguage(codeLanguage: CodeLanguage) { - cancelAllRunningTasks() - state.setLanguage(codeLanguage: codeLanguage) } // MARK: - HighlightProviding - /// Set up and parse the initial language tree and all injected layers. - func setUp(textView: HighlighterTextView) { + /// Set up the client with a text view and language. + /// - Parameters: + /// - textView: The text view to use as a data source. + /// A weak reference will be kept for the lifetime of this object. + /// - codeLanguage: The language to use for parsing. + func setUp(textView: HighlighterTextView, codeLanguage: CodeLanguage) { cancelAllRunningTasks() - + queueLock.lock() self.textView = textView - readBlock = textView.createReadBlock() - - state.setUp(textView: textView) + self.readBlock = textView.createReadBlock() + queuedEdits.append { + self.stateLock.lock() + if self.state == nil { + self.state = TreeSitterState(codeLanguage: codeLanguage, textView: textView) + } else { + self.state?.setLanguage(codeLanguage) + } + self.stateLock.unlock() + } + beginTasksIfNeeded() + queueLock.unlock() } /// Notifies the highlighter of an edit and in exchange gets a set of indices that need to be re-highlighted. @@ -104,7 +118,6 @@ final class TreeSitterClient: HighlightProviding { delta: Int, completion: @escaping ((IndexSet) -> Void) ) { - print("Received Edit", range, delta) guard let edit = InputEdit(range: range, delta: delta, oldEndPoint: .zero) else { return } queueLock.lock() @@ -112,12 +125,10 @@ final class TreeSitterClient: HighlightProviding { let longDocument = textView.documentRange.length > Constants.maxSyncContentLength if hasOutstandingWork || longEdit || longDocument { - print("\tQueueing Async") applyEditAsync(editState: EditState(edit: edit, completion: completion), startAtLayerIndex: 0) queueLock.unlock() } else { queueLock.unlock() - print("Performing Sync") applyEdit(editState: EditState(edit: edit, completion: completion)) } } @@ -132,24 +143,22 @@ final class TreeSitterClient: HighlightProviding { range: NSRange, completion: @escaping (([HighlightRange]) -> Void) ) { - print("Received Query", range) queueLock.lock() let longQuery = range.length > Constants.maxSyncQueryLength let longDocument = textView.documentRange.length > Constants.maxSyncContentLength if hasOutstandingWork || longQuery || longDocument { - print("\tQueuing Async Query") queryHighlightsForRangeAsync(range: range, completion: completion) queueLock.unlock() } else { queueLock.unlock() - print("\tQueuing Sync Query") queryHighlightsForRange(range: range, runningAsync: false, completion: completion) } } // MARK: - Async Helpers + /// Use to determine if there are any queued or running async tasks. var hasOutstandingWork: Bool { runningTask != nil || queuedEdits.count > 0 || queuedQueries.count > 0 } @@ -162,21 +171,8 @@ final class TreeSitterClient: HighlightProviding { self.runningTask = nil } - var info = mach_timebase_info() - guard mach_timebase_info(&info) == KERN_SUCCESS else { return } - - let start = mach_absolute_time() - do { while let nextQueuedTask = self.determineNextTask() { - let end = mach_absolute_time() - - let elapsed = end - start - - let nanos = elapsed * UInt64(info.numer) / UInt64(info.denom) - - print("\t\tFound Next Task. Tasks Remaining:", self.queuedEdits.count + self.queuedQueries.count) - print("\t\tTime Since queue started (ms): ", TimeInterval(nanos) / TimeInterval(NSEC_PER_MSEC)) try Task.checkCancellation() nextQueuedTask() } @@ -203,8 +199,10 @@ final class TreeSitterClient: HighlightProviding { /// Cancels all running and enqueued tasks. private func cancelAllRunningTasks() { + queueLock.lock() runningTask?.cancel() queuedEdits.removeAll() queuedQueries.removeAll() + queueLock.unlock() } } diff --git a/Sources/CodeEditTextView/TreeSitter/TreeSitterState.swift b/Sources/CodeEditTextView/TreeSitter/TreeSitterState.swift index 4f5a35552..74165b36d 100644 --- a/Sources/CodeEditTextView/TreeSitter/TreeSitterState.swift +++ b/Sources/CodeEditTextView/TreeSitter/TreeSitterState.swift @@ -17,19 +17,14 @@ class TreeSitterState { // MARK: - Init - init(primaryLayer: TreeSitterLanguage) { - self.primaryLayer = primaryLayer - } + init(codeLanguage: CodeLanguage, textView: HighlighterTextView) { + self.primaryLayer = codeLanguage.id - /// Initialize the state object with the given read block. - /// - Parameter textView: The textView to use as a data source. - public func setUp(textView: HighlighterTextView) { + self.setLanguage(codeLanguage) let readBlock = textView.createReadBlock() - let previousTimeout = layers[0].parser.timeout layers[0].parser.timeout = 0.0 layers[0].tree = layers[0].parser.createTree(readBlock: readBlock) - layers[0].parser.timeout = previousTimeout var layerSet = Set(arrayLiteral: layers[0]) var touchedLayers = Set() @@ -47,10 +42,7 @@ class TreeSitterState { } } - /// Sets the primary language for the client. Will reset all layers, will not do any parsing work. - /// - Parameter codeLanguage: The new primary language. - public func setLanguage(codeLanguage: CodeLanguage) { - // Remove all trees and languages, everything needs to be re-parsed. + public func setLanguage(_ codeLanguage: CodeLanguage) { layers.removeAll() primaryLayer = codeLanguage.id @@ -65,9 +57,8 @@ class TreeSitterState { ) ] - if let treeSitterLanguage = codeLanguage.language { - try? layers[0].parser.setLanguage(treeSitterLanguage) - } + guard let treeSitterLanguage = codeLanguage.language else { return } + try? layers[0].parser.setLanguage(treeSitterLanguage) } // MARK: - Layer Management From 1f72713a93064e1d3d97fde2fd7e159ffe24e6b6 Mon Sep 17 00:00:00 2001 From: Khan Winter <35942988+thecoolwinter@users.noreply.github.com> Date: Sat, 3 Jun 2023 00:04:29 -0500 Subject: [PATCH 03/10] Use `PThreadLock`, Add Parallel Highlights --- Package.resolved | 8 +-- .../CodeEditTextView/Enums/CaptureName.swift | 2 +- .../STTextView+HighlighterTextView.swift | 21 ++++++++ .../STTextView+/STTextView+VisibleRange.swift | 27 ++++++++-- .../Highlighting/HighlighterTextView.swift | 12 ----- ...anguageLayer.swift => LanguageLayer.swift} | 17 ++++-- .../TreeSitter/PthreadLock.swift | 29 +++++++++++ .../TreeSitter/TreeSitterClient+Edit.swift | 6 +-- .../TreeSitterClient+Highlight.swift | 7 +-- .../TreeSitter/TreeSitterClient.swift | 52 ++++++++++++++----- .../TreeSitter/TreeSitterState.swift | 43 +++++++++------ 11 files changed, 164 insertions(+), 60 deletions(-) create mode 100644 Sources/CodeEditTextView/Extensions/STTextView+/STTextView+HighlighterTextView.swift rename Sources/CodeEditTextView/TreeSitter/{TSLanguageLayer.swift => LanguageLayer.swift} (90%) create mode 100644 Sources/CodeEditTextView/TreeSitter/PthreadLock.swift diff --git a/Package.resolved b/Package.resolved index f58b470f3..c959a67c5 100644 --- a/Package.resolved +++ b/Package.resolved @@ -5,8 +5,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/CodeEditApp/CodeEditLanguages.git", "state" : { - "revision" : "08cb9dc04e70d1e7b9610580794ed4da774c2b84", - "version" : "0.1.13" + "revision" : "016b0e8df9603122202aef1f17cdc7fb2b51e1d0", + "version" : "0.1.14" } }, { @@ -23,8 +23,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/krzyzanowskim/STTextView", "state" : { - "revision" : "41c7c87a552e6286ebea5f348e820b529c6f5662", - "version" : "0.4.2" + "revision" : "6c6243ae790247368aea258e4fb3642523759bec", + "version" : "0.5.3" } }, { diff --git a/Sources/CodeEditTextView/Enums/CaptureName.swift b/Sources/CodeEditTextView/Enums/CaptureName.swift index 8fc15c9b3..9c8599cad 100644 --- a/Sources/CodeEditTextView/Enums/CaptureName.swift +++ b/Sources/CodeEditTextView/Enums/CaptureName.swift @@ -33,7 +33,7 @@ public enum CaptureName: String, CaseIterable { /// - Parameter string: A string to get the capture name from /// - Returns: A `CaptureNames` case static func fromString(_ string: String?) -> CaptureName? { - allCases.first { $0.rawValue == string } + CaptureName(rawValue: string ?? "") } var alternate: CaptureName { diff --git a/Sources/CodeEditTextView/Extensions/STTextView+/STTextView+HighlighterTextView.swift b/Sources/CodeEditTextView/Extensions/STTextView+/STTextView+HighlighterTextView.swift new file mode 100644 index 000000000..9b2b005ee --- /dev/null +++ b/Sources/CodeEditTextView/Extensions/STTextView+/STTextView+HighlighterTextView.swift @@ -0,0 +1,21 @@ +// +// STTextView+HighlighterTextView.swift +// CodeEditTextView +// +// Created by Khan Winter on 6/2/23. +// + +import Foundation +import STTextView + +/// A default implementation for `STTextView` to be passed to `HighlightProviding` objects. +extension STTextView: HighlighterTextView { + public var documentRange: NSRange { + return NSRange(location: 0, + length: textContentStorage?.textStorage?.length ?? 0) + } + + public func stringForRange(_ nsRange: NSRange) -> String? { + return textContentStorage?.textStorage?.mutableString.substring(with: nsRange) + } +} diff --git a/Sources/CodeEditTextView/Extensions/STTextView+/STTextView+VisibleRange.swift b/Sources/CodeEditTextView/Extensions/STTextView+/STTextView+VisibleRange.swift index 722017cf5..0d366ab0e 100644 --- a/Sources/CodeEditTextView/Extensions/STTextView+/STTextView+VisibleRange.swift +++ b/Sources/CodeEditTextView/Extensions/STTextView+/STTextView+VisibleRange.swift @@ -26,15 +26,34 @@ extension STTextView { // Calculate min & max points w/ a small amount of padding vertically. let minPoint = CGPoint(x: bounds.minX, - y: bounds.minY - 100) + y: bounds.minY - 200) let maxPoint = CGPoint(x: bounds.maxX, - y: bounds.maxY + 100) + y: bounds.maxY + 200) // Get text fragments for both the min and max points - guard let start = textLayoutManager.textLayoutFragment(for: minPoint)?.rangeInElement.location, - let end = textLayoutManager.textLayoutFragment(for: maxPoint)?.rangeInElement.endLocation else { + guard let start = textLayoutManager.textLayoutFragment(for: minPoint)?.rangeInElement.location else { return nil } + + // End point can be tricky sometimes. If the document is smaller than the scroll view it can sometimes return + // nil for the `maxPoint` layout fragment. So we attempt to grab the last fragment. + var end: NSTextLocation? + + if let endFragment = textLayoutManager.textLayoutFragment(for: maxPoint) { + end = endFragment.rangeInElement.location + } else { + textLayoutManager.ensureLayout(for: NSTextRange(location: textLayoutManager.documentRange.endLocation)) + textLayoutManager.enumerateTextLayoutFragments( + from: textLayoutManager.documentRange.endLocation, + options: [.reverse, .ensuresLayout, .ensuresExtraLineFragment] + ) { layoutFragment in + end = layoutFragment.rangeInElement.endLocation + return false + } + } + + guard let end else { return nil } + guard start.compare(end) != .orderedDescending else { return NSTextRange(location: end, end: start)?.nsRange(using: textContentStorage) } diff --git a/Sources/CodeEditTextView/Highlighting/HighlighterTextView.swift b/Sources/CodeEditTextView/Highlighting/HighlighterTextView.swift index a79755375..ff6b2ac7f 100644 --- a/Sources/CodeEditTextView/Highlighting/HighlighterTextView.swift +++ b/Sources/CodeEditTextView/Highlighting/HighlighterTextView.swift @@ -16,15 +16,3 @@ public protocol HighlighterTextView: AnyObject { /// A substring for the requested range. func stringForRange(_ nsRange: NSRange) -> String? } - -/// A default implementation for `STTextView` to be passed to `HighlightProviding` objects. -extension STTextView: HighlighterTextView { - public var documentRange: NSRange { - return NSRange(location: 0, - length: textContentStorage?.textStorage?.length ?? 0) - } - - public func stringForRange(_ nsRange: NSRange) -> String? { - return textContentStorage?.textStorage?.mutableString.substring(with: nsRange) - } -} diff --git a/Sources/CodeEditTextView/TreeSitter/TSLanguageLayer.swift b/Sources/CodeEditTextView/TreeSitter/LanguageLayer.swift similarity index 90% rename from Sources/CodeEditTextView/TreeSitter/TSLanguageLayer.swift rename to Sources/CodeEditTextView/TreeSitter/LanguageLayer.swift index 9bdb4b552..0c12c688b 100644 --- a/Sources/CodeEditTextView/TreeSitter/TSLanguageLayer.swift +++ b/Sources/CodeEditTextView/TreeSitter/LanguageLayer.swift @@ -9,7 +9,7 @@ import Foundation import CodeEditLanguages import SwiftTreeSitter -class TSLanguageLayer: Hashable { +public class LanguageLayer: Hashable { /// Initialize a language layer /// - Parameters: /// - id: The ID of the layer. @@ -43,11 +43,22 @@ class TSLanguageLayer: Hashable { var languageQuery: Query? var ranges: [NSRange] - static func == (lhs: TSLanguageLayer, rhs: TSLanguageLayer) -> Bool { + func copy() -> LanguageLayer { + return LanguageLayer( + id: id, + parser: parser, + supportsInjections: supportsInjections, + tree: tree?.copy(), + languageQuery: languageQuery, + ranges: ranges + ) + } + + public static func == (lhs: LanguageLayer, rhs: LanguageLayer) -> Bool { return lhs.id == rhs.id && lhs.ranges == rhs.ranges } - func hash(into hasher: inout Hasher) { + public func hash(into hasher: inout Hasher) { hasher.combine(id) hasher.combine(ranges) } diff --git a/Sources/CodeEditTextView/TreeSitter/PthreadLock.swift b/Sources/CodeEditTextView/TreeSitter/PthreadLock.swift new file mode 100644 index 000000000..60ecc5e01 --- /dev/null +++ b/Sources/CodeEditTextView/TreeSitter/PthreadLock.swift @@ -0,0 +1,29 @@ +// +// PthreadLock.swift +// CodeEditTextView +// +// Created by Khan Winter on 6/2/23. +// + +import Foundation + +/// A thread safe, atomic lock that wraps a `pthread_mutex_t` +struct PthreadLock { + private var _lock: pthread_mutex_t + + /// Initializes the lock + init() { + _lock = .init() + pthread_mutex_init(&_lock, nil) + } + + /// Locks the lock, if the lock is already locked it will block the current thread until it unlocks. + mutating func lock() { + pthread_mutex_lock(&_lock) + } + + /// Unlocks the lock. + mutating func unlock() { + pthread_mutex_unlock(&_lock) + } +} diff --git a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+Edit.swift b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+Edit.swift index 7f9238840..fae473040 100644 --- a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+Edit.swift +++ b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+Edit.swift @@ -14,8 +14,8 @@ extension TreeSitterClient { class EditState { var edit: InputEdit var rangeSet: IndexSet - var layerSet: Set - var touchedLayers: Set + var layerSet: Set + var touchedLayers: Set var completion: ((IndexSet) -> Void) init( @@ -46,7 +46,7 @@ extension TreeSitterClient { for layerIdx in (startIdx.. Void) ) { - guard let textView, let state else { return } stateLock.lock() + guard let textView, let state = state?.copy() else { return } + stateLock.unlock() var highlights: [HighlightRange] = [] var injectedSet = IndexSet(integersIn: range) - for layer in state.layers where layer.id != state.primaryLayer { + for layer in state.layers where layer.id != state.primaryLayer.id { // Query injected only if a layer's ranges intersects with `range` for layerRange in layer.ranges { if let rangeIntersection = range.intersection(layerRange) { @@ -72,7 +73,7 @@ extension TreeSitterClient { /// - range: The range to query for. /// - Returns: Any ranges to highlight. internal func queryLayerHighlights( - layer: TSLanguageLayer, + layer: LanguageLayer, textView: HighlighterTextView, range: NSRange ) -> [HighlightRange] { diff --git a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift index d9cf87c0a..f77020ab4 100644 --- a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift +++ b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift @@ -6,7 +6,6 @@ // import Foundation -import os import CodeEditLanguages import SwiftTreeSitter @@ -17,10 +16,10 @@ import SwiftTreeSitter /// Use the `init` method to set up the client initially. If text changes it should be able to be read through the /// `textProvider` callback. You can optionally update the text manually using the `setText` method. /// However, the `setText` method will re-compile the entire corpus so should be used sparingly. -final class TreeSitterClient: HighlightProviding { +public final class TreeSitterClient: HighlightProviding { typealias AsyncCallback = @Sendable () -> Void - // MARK: - Properties/Constants + // MARK: - Properties public var identifier: String { "CodeEdit.TreeSitterClient" @@ -39,14 +38,16 @@ final class TreeSitterClient: HighlightProviding { internal var queuedQueries: [AsyncCallback] = [] /// A lock that must be obtained whenever `state` is modified - internal var stateLock: OSAllocatedUnfairLock = OSAllocatedUnfairLock() + internal var stateLock: PthreadLock = PthreadLock() /// A lock that must be obtained whenever either `queuedEdits` or `queuedHighlights` is modified - internal var queueLock: OSAllocatedUnfairLock = OSAllocatedUnfairLock() + internal var queueLock: PthreadLock = PthreadLock() /// The internal tree-sitter layer tree object. internal var state: TreeSitterState? internal var textProvider: ResolvingQueryCursor.TextProvider + // MARK: - Constants + internal enum Constants { /// The maximum amount of limits a cursor can match during a query. /// Used to ensure performance in large files, even though we generally limit the query to the visible range. @@ -67,6 +68,9 @@ final class TreeSitterClient: HighlightProviding { /// The maximum length a query can be before it must be performed asynchronously. static let maxSyncQueryLength: Int = 4096 + + /// The maximum number of highlight queries that can be performed in parallel. + static let simultaneousHighlightLimit: Int = 5 } // MARK: - Init/Config @@ -87,7 +91,7 @@ final class TreeSitterClient: HighlightProviding { /// - textView: The text view to use as a data source. /// A weak reference will be kept for the lifetime of this object. /// - codeLanguage: The language to use for parsing. - func setUp(textView: HighlighterTextView, codeLanguage: CodeLanguage) { + public func setUp(textView: HighlighterTextView, codeLanguage: CodeLanguage) { cancelAllRunningTasks() queueLock.lock() self.textView = textView @@ -112,7 +116,7 @@ final class TreeSitterClient: HighlightProviding { /// - range: The range of the edit. /// - delta: The length of the edit, can be negative for deletions. /// - completion: The function to call with an `IndexSet` containing all Indices to invalidate. - func applyEdit( + public func applyEdit( textView: HighlighterTextView, range: NSRange, delta: Int, @@ -138,7 +142,7 @@ final class TreeSitterClient: HighlightProviding { /// - textView: The text view to use. /// - range: The range to limit the highlights to. /// - completion: Called when the query completes. - func queryHighlightsFor( + public func queryHighlightsFor( textView: HighlighterTextView, range: NSRange, completion: @escaping (([HighlightRange]) -> Void) @@ -156,14 +160,21 @@ final class TreeSitterClient: HighlightProviding { } } - // MARK: - Async Helpers + // MARK: - Async /// Use to determine if there are any queued or running async tasks. var hasOutstandingWork: Bool { runningTask != nil || queuedEdits.count > 0 || queuedQueries.count > 0 } + private enum QueuedTaskType { + case edit(job: AsyncCallback) + case highlight(jobs: [AsyncCallback]) + } + /// Spawn the running task if one is needed and doesn't already exist. + /// + /// The task will run until `determineNextTask` returns nil. It will run any highlight jobs in parallel. internal func beginTasksIfNeeded() { guard runningTask == nil && (queuedEdits.count > 0 || queuedQueries.count > 0) else { return } runningTask = Task.detached(priority: .userInitiated) { @@ -174,14 +185,26 @@ final class TreeSitterClient: HighlightProviding { do { while let nextQueuedTask = self.determineNextTask() { try Task.checkCancellation() - nextQueuedTask() + switch nextQueuedTask { + case .edit(let job): + job() + case .highlight(let jobs): + await withTaskGroup(of: Void.self, body: { taskGroup in + for job in jobs { + taskGroup.addTask { + job() + } + } + }) + } } } catch { } } } /// Determines the next async task to run and returns it if it exists. - private func determineNextTask() -> AsyncCallback? { + /// Greedily returns queued highlight jobs determined by `Constants.simultaneousHighlightLimit` + private func determineNextTask() -> QueuedTaskType? { queueLock.lock() defer { queueLock.unlock() @@ -189,9 +212,12 @@ final class TreeSitterClient: HighlightProviding { // Get an edit task if any, otherwise get a highlight task if any. if queuedEdits.count > 0 { - return queuedEdits.removeFirst() + return .edit(job: queuedEdits.removeFirst()) } else if queuedQueries.count > 0 { - return queuedQueries.removeFirst() + let jobCount = min(queuedQueries.count, Constants.simultaneousHighlightLimit) + let jobs = Array(queuedQueries[0..(arrayLiteral: layers[0]) - var touchedLayers = Set() + var layerSet = Set(arrayLiteral: layers[0]) + var touchedLayers = Set() var idx = 0 while idx < layers.count { @@ -42,12 +42,17 @@ class TreeSitterState { } } + private init(codeLanguage: CodeLanguage, layers: [LanguageLayer]) { + self.primaryLayer = codeLanguage + self.layers = layers + } + public func setLanguage(_ codeLanguage: CodeLanguage) { layers.removeAll() - primaryLayer = codeLanguage.id + primaryLayer = codeLanguage layers = [ - TSLanguageLayer( + LanguageLayer( id: codeLanguage.id, parser: Parser(), supportsInjections: codeLanguage.additionalHighlights?.contains("injections") ?? false, @@ -61,6 +66,10 @@ class TreeSitterState { try? layers[0].parser.setLanguage(treeSitterLanguage) } + public func copy() -> TreeSitterState { + return TreeSitterState(codeLanguage: primaryLayer, layers: layers.map { $0.copy() }) + } + // MARK: - Layer Management /// Removes a layer at the given index. @@ -69,7 +78,7 @@ class TreeSitterState { layers.remove(at: idx) } - public func removeLanguageLayers(in set: Set) { + public func removeLanguageLayers(in set: Set) { layers.removeAll(where: { set.contains($0 )}) } @@ -81,14 +90,14 @@ class TreeSitterState { public func addLanguageLayer( layerId: TreeSitterLanguage, readBlock: @escaping Parser.ReadBlock - ) -> TSLanguageLayer? { + ) -> LanguageLayer? { guard let language = CodeLanguage.allLanguages.first(where: { $0.id == layerId }), let parserLanguage = language.language else { return nil } - let newLayer = TSLanguageLayer( + let newLayer = LanguageLayer( id: layerId, parser: Parser(), supportsInjections: language.additionalHighlights?.contains("injections") ?? false, @@ -118,7 +127,7 @@ class TreeSitterState { /// - Returns: A set of indices of any new layers. This set indicates ranges that should be re-highlighted. public func updateInjectedLayers( textView: HighlighterTextView, - touchedLayers: Set + touchedLayers: Set ) -> IndexSet { var layerSet = Set(layers) var touchedLayers = touchedLayers @@ -163,9 +172,9 @@ class TreeSitterState { @discardableResult private func updateInjectedLanguageLayer( textView: HighlighterTextView, - layer: TSLanguageLayer, - layerSet: inout Set, - touchedLayers: inout Set + layer: LanguageLayer, + layerSet: inout Set, + touchedLayers: inout Set ) -> IndexSet { guard let tree = layer.tree, let rootNode = tree.rootNode, @@ -186,13 +195,13 @@ class TreeSitterState { continue } - if treeSitterLanguage == primaryLayer { + if treeSitterLanguage == primaryLayer.id { continue } for range in ranges { // Temp layer object - let layer = TSLanguageLayer( + let layer = LanguageLayer( id: treeSitterLanguage, parser: Parser(), supportsInjections: false, From 6392429f3590b5d4230074e3515ed18903970fb1 Mon Sep 17 00:00:00 2001 From: Khan Winter <35942988+thecoolwinter@users.noreply.github.com> Date: Sat, 3 Jun 2023 00:50:16 -0500 Subject: [PATCH 04/10] Correctly Destroy `pthread_mutex_t` --- Sources/CodeEditTextView/TreeSitter/PthreadLock.swift | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Sources/CodeEditTextView/TreeSitter/PthreadLock.swift b/Sources/CodeEditTextView/TreeSitter/PthreadLock.swift index 60ecc5e01..739fb5c8a 100644 --- a/Sources/CodeEditTextView/TreeSitter/PthreadLock.swift +++ b/Sources/CodeEditTextView/TreeSitter/PthreadLock.swift @@ -8,7 +8,7 @@ import Foundation /// A thread safe, atomic lock that wraps a `pthread_mutex_t` -struct PthreadLock { +class PthreadLock { private var _lock: pthread_mutex_t /// Initializes the lock @@ -18,12 +18,16 @@ struct PthreadLock { } /// Locks the lock, if the lock is already locked it will block the current thread until it unlocks. - mutating func lock() { + func lock() { pthread_mutex_lock(&_lock) } /// Unlocks the lock. - mutating func unlock() { + func unlock() { pthread_mutex_unlock(&_lock) } + + deinit { + pthread_mutex_destroy(&_lock) + } } From 42e501a95acb794b734a8d203969c1daae3279c0 Mon Sep 17 00:00:00 2001 From: Khan Winter <35942988+thecoolwinter@users.noreply.github.com> Date: Sat, 3 Jun 2023 01:17:23 -0500 Subject: [PATCH 05/10] Implement Basic Tests --- .../TreeSitter/TreeSitterClient.swift | 4 + .../TreeSitterClientTests.swift | 109 ++++++++++++++++++ 2 files changed, 113 insertions(+) create mode 100644 Tests/CodeEditTextViewTests/TreeSitterClientTests.swift diff --git a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift index f77020ab4..9ec7d9b69 100644 --- a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift +++ b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift @@ -231,4 +231,8 @@ public final class TreeSitterClient: HighlightProviding { queuedQueries.removeAll() queueLock.unlock() } + + deinit { + cancelAllRunningTasks() + } } diff --git a/Tests/CodeEditTextViewTests/TreeSitterClientTests.swift b/Tests/CodeEditTextViewTests/TreeSitterClientTests.swift new file mode 100644 index 000000000..ce5fe15f3 --- /dev/null +++ b/Tests/CodeEditTextViewTests/TreeSitterClientTests.swift @@ -0,0 +1,109 @@ +import XCTest +@testable import CodeEditTextView + +fileprivate class TestTextView: HighlighterTextView { + var testString: NSMutableString = "func testSwiftFunc() -> Int {\n\tprint(\"\")\n}" + + var documentRange: NSRange { + NSRange(location: 0, length: testString.length) + } + + func stringForRange(_ nsRange: NSRange) -> String? { + testString.substring(with: nsRange) + } +} + +// swiftlint:disable all +final class TreeSitterClientTests: XCTestCase { + + fileprivate var textView = TestTextView() + var client: TreeSitterClient! + + override func setUp() { + client = TreeSitterClient { nsRange, _ in + self.textView.stringForRange(nsRange) + } + } + + func test_clientSetup() { + client.setUp(textView: textView, codeLanguage: .swift) + XCTAssert(client.hasOutstandingWork, "Client should queue language loading on a background task.") + + let editExpectation = expectation(description: "Edit work should never return") + editExpectation.isInverted = true // Expect to never happen + + textView.testString.insert("let int = 0\n", at: 0) + client.applyEdit(textView: textView, range: NSRange(location: 0, length: 12), delta: 12) { _ in + editExpectation.fulfill() + } + + client.setUp(textView: textView, codeLanguage: .swift) + XCTAssert(client.hasOutstandingWork, "Client should queue language loading on a background task.") + XCTAssert(client.queuedEdits.count == 1, "Client should cancel all queued work when setUp is called.") + + waitForExpectations(timeout: 1.0, handler: nil) + } + + // Test async language loading with edits and highlights queued before loading completes. + func test_languageLoad() { + textView = TestTextView() + client.setUp(textView: textView, codeLanguage: .swift) + + XCTAssert(client.hasOutstandingWork, "Client should queue language loading on a background task.") + + let editExpectation = expectation(description: "Edit work should return first.") + let highlightExpectation = expectation(description: "Highlight should return last.") + + client.queryHighlightsFor(textView: textView, range: NSRange(location: 0, length: 42)) { _ in + highlightExpectation.fulfill() + } + + textView.testString.insert("let int = 0\n", at: 0) + client.applyEdit(textView: textView, range: NSRange(location: 0, length: 12), delta: 12) { _ in + editExpectation.fulfill() + } + + wait(for: [editExpectation, highlightExpectation], timeout: 10.0, enforceOrder: true) + } + + // Edits should be consumed before highlights. + func test_queueOrder() { + textView = TestTextView() + client.setUp(textView: textView, codeLanguage: .swift) + + let editExpectation = expectation(description: "Edit work should return first.") + let editExpectation2 = expectation(description: "Edit2 should return 2nd.") + let highlightExpectation = expectation(description: "Highlight should return 3rd.") + + // Do initial query while language loads. + client.queryHighlightsFor(textView: textView, range: NSRange(location: 0, length: 42)) { _ in + print("highlightExpectation") + highlightExpectation.fulfill() + } + + // Queue another edit + textView.testString.insert("let int = 0\n", at: 0) + client.applyEdit(textView: textView, range: NSRange(location: 0, length: 12), delta: 12) { _ in + print("editExpectation") + editExpectation.fulfill() + } + + // One more edit + textView.testString.insert("let int = 0\n", at: 0) + client.applyEdit(textView: textView, range: NSRange(location: 0, length: 12), delta: 12) { _ in + print("editExpectation2") + editExpectation2.fulfill() + } + + wait( + for: [ + editExpectation, + editExpectation2, + highlightExpectation, + ], + timeout: 10.0, + enforceOrder: true + ) + } +} +// swiftlint:enable all From de8ea812ef9100064464a728ad5d339eb62b1314 Mon Sep 17 00:00:00 2001 From: Khan Winter <35942988+thecoolwinter@users.noreply.github.com> Date: Sat, 3 Jun 2023 01:30:45 -0500 Subject: [PATCH 06/10] Rename `determineNextTask` to `determineNextJob` --- .../CodeEditTextView/TreeSitter/TreeSitterClient.swift | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift index 9ec7d9b69..8d891ba67 100644 --- a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift +++ b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift @@ -183,9 +183,9 @@ public final class TreeSitterClient: HighlightProviding { } do { - while let nextQueuedTask = self.determineNextTask() { + while let nextQueuedJob = self.determineNextJob() { try Task.checkCancellation() - switch nextQueuedTask { + switch nextQueuedJob { case .edit(let job): job() case .highlight(let jobs): @@ -202,9 +202,9 @@ public final class TreeSitterClient: HighlightProviding { } } - /// Determines the next async task to run and returns it if it exists. + /// Determines the next async job to run and returns it if it exists. /// Greedily returns queued highlight jobs determined by `Constants.simultaneousHighlightLimit` - private func determineNextTask() -> QueuedTaskType? { + private func determineNextJob() -> QueuedTaskType? { queueLock.lock() defer { queueLock.unlock() From 9ef99bc98b130e0fe3ff5eced8a932893275b788 Mon Sep 17 00:00:00 2001 From: Khan Winter <35942988+thecoolwinter@users.noreply.github.com> Date: Sat, 3 Jun 2023 01:45:04 -0500 Subject: [PATCH 07/10] Remove Debug Comment --- .../Filters/STTextViewController+TextFormation.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Sources/CodeEditTextView/Filters/STTextViewController+TextFormation.swift b/Sources/CodeEditTextView/Filters/STTextViewController+TextFormation.swift index 67fb03e52..5e24773ed 100644 --- a/Sources/CodeEditTextView/Filters/STTextViewController+TextFormation.swift +++ b/Sources/CodeEditTextView/Filters/STTextViewController+TextFormation.swift @@ -107,11 +107,11 @@ extension STTextViewController { range: range, limit: textView.textContentStorage?.length ?? 0) -// textView.undoManager?.beginUndoGrouping() + textView.undoManager?.beginUndoGrouping() let result = shouldApplyMutation(mutation, to: textView) -// textView.undoManager?.endUndoGrouping() + textView.undoManager?.endUndoGrouping() return result } From 02c3162990f621a8cb4c6e71adf3ddd8dd0f685b Mon Sep 17 00:00:00 2001 From: Khan Winter <35942988+thecoolwinter@users.noreply.github.com> Date: Sat, 3 Jun 2023 01:50:25 -0500 Subject: [PATCH 08/10] Add Documentation --- .../TreeSitter/TreeSitterState.swift | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/Sources/CodeEditTextView/TreeSitter/TreeSitterState.swift b/Sources/CodeEditTextView/TreeSitter/TreeSitterState.swift index 401ef016e..e48b9b019 100644 --- a/Sources/CodeEditTextView/TreeSitter/TreeSitterState.swift +++ b/Sources/CodeEditTextView/TreeSitter/TreeSitterState.swift @@ -10,13 +10,16 @@ import SwiftTreeSitter import CodeEditLanguages /// TreeSitterState contains the tree of language layers that make up the tree-sitter document. -/// Use the given public class TreeSitterState { private(set) var primaryLayer: CodeLanguage private(set) var layers: [LanguageLayer] = [] // MARK: - Init + /// Initialize a state object with a language and text view. + /// - Parameters: + /// - codeLanguage: The language to use. + /// - textView: The text view to use as a text data source. init(codeLanguage: CodeLanguage, textView: HighlighterTextView) { self.primaryLayer = codeLanguage @@ -42,11 +45,14 @@ public class TreeSitterState { } } + /// Private initilizer used by `copy` private init(codeLanguage: CodeLanguage, layers: [LanguageLayer]) { self.primaryLayer = codeLanguage self.layers = layers } + /// Sets the language for the state. Removing all existing layers. + /// - Parameter codeLanguage: The language to use. public func setLanguage(_ codeLanguage: CodeLanguage) { layers.removeAll() @@ -66,6 +72,8 @@ public class TreeSitterState { try? layers[0].parser.setLanguage(treeSitterLanguage) } + /// Creates a copy of this state object. + /// - Returns: The copied object public func copy() -> TreeSitterState { return TreeSitterState(codeLanguage: primaryLayer, layers: layers.map { $0.copy() }) } @@ -78,6 +86,8 @@ public class TreeSitterState { layers.remove(at: idx) } + /// Removes all languagel ayers in the given set. + /// - Parameter set: A set of all language layers to remove. public func removeLanguageLayers(in set: Set) { layers.removeAll(where: { set.contains($0 )}) } From 7705e9145773f01bfda74c36dfba48c216ae23d0 Mon Sep 17 00:00:00 2001 From: Khan Winter <35942988+thecoolwinter@users.noreply.github.com> Date: Sat, 3 Jun 2023 01:51:46 -0500 Subject: [PATCH 09/10] Lint Error --- Tests/CodeEditTextViewTests/TreeSitterClientTests.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/CodeEditTextViewTests/TreeSitterClientTests.swift b/Tests/CodeEditTextViewTests/TreeSitterClientTests.swift index ce5fe15f3..a12580492 100644 --- a/Tests/CodeEditTextViewTests/TreeSitterClientTests.swift +++ b/Tests/CodeEditTextViewTests/TreeSitterClientTests.swift @@ -1,6 +1,7 @@ import XCTest @testable import CodeEditTextView +// swiftlint:disable all fileprivate class TestTextView: HighlighterTextView { var testString: NSMutableString = "func testSwiftFunc() -> Int {\n\tprint(\"\")\n}" @@ -13,7 +14,6 @@ fileprivate class TestTextView: HighlighterTextView { } } -// swiftlint:disable all final class TreeSitterClientTests: XCTestCase { fileprivate var textView = TestTextView() From deb671f5fa0e2f05236999189501bbda99c5e90d Mon Sep 17 00:00:00 2001 From: Khan Winter <35942988+thecoolwinter@users.noreply.github.com> Date: Sat, 3 Jun 2023 10:32:04 -0500 Subject: [PATCH 10/10] Fix Formatting --- .../STTextView+/STTextView+HighlighterTextView.swift | 6 ++++-- .../STTextView+/STTextView+VisibleRange.swift | 12 ++++++++---- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/Sources/CodeEditTextView/Extensions/STTextView+/STTextView+HighlighterTextView.swift b/Sources/CodeEditTextView/Extensions/STTextView+/STTextView+HighlighterTextView.swift index 9b2b005ee..a9e7b0cf6 100644 --- a/Sources/CodeEditTextView/Extensions/STTextView+/STTextView+HighlighterTextView.swift +++ b/Sources/CodeEditTextView/Extensions/STTextView+/STTextView+HighlighterTextView.swift @@ -11,8 +11,10 @@ import STTextView /// A default implementation for `STTextView` to be passed to `HighlightProviding` objects. extension STTextView: HighlighterTextView { public var documentRange: NSRange { - return NSRange(location: 0, - length: textContentStorage?.textStorage?.length ?? 0) + return NSRange( + location: 0, + length: textContentStorage?.textStorage?.length ?? 0 + ) } public func stringForRange(_ nsRange: NSRange) -> String? { diff --git a/Sources/CodeEditTextView/Extensions/STTextView+/STTextView+VisibleRange.swift b/Sources/CodeEditTextView/Extensions/STTextView+/STTextView+VisibleRange.swift index 0d366ab0e..b3a973025 100644 --- a/Sources/CodeEditTextView/Extensions/STTextView+/STTextView+VisibleRange.swift +++ b/Sources/CodeEditTextView/Extensions/STTextView+/STTextView+VisibleRange.swift @@ -25,10 +25,14 @@ extension STTextView { } // Calculate min & max points w/ a small amount of padding vertically. - let minPoint = CGPoint(x: bounds.minX, - y: bounds.minY - 200) - let maxPoint = CGPoint(x: bounds.maxX, - y: bounds.maxY + 200) + let minPoint = CGPoint( + x: bounds.minX, + y: bounds.minY - 200 + ) + let maxPoint = CGPoint( + x: bounds.maxX, + y: bounds.maxY + 200 + ) // Get text fragments for both the min and max points guard let start = textLayoutManager.textLayoutFragment(for: minPoint)?.rangeInElement.location else {