Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Another search speed up 💨 #1500

Merged
merged 10 commits into from
Dec 17, 2023
Merged

Another search speed up 💨 #1500

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
8076db8
Added regX patters instead of normal file search
tom-ludwig Dec 4, 2023
6deda98
Update SearchIndexer+InternalMethods.swift
tom-ludwig Dec 5, 2023
3e8cb97
Performance update
tom-ludwig Dec 5, 2023
c6f34b0
fix: out of bounds
tom-ludwig Dec 7, 2023
344f56b
That ain't working :(
tom-ludwig Dec 8, 2023
f8abb65
Fixed line previews for new search function
tom-ludwig Dec 12, 2023
b10f631
Split the evaluateFile function into smaller, focused functions
tom-ludwig Dec 12, 2023
37ffb85
Merge branch 'main' into search-speedUP-2.0
tom-ludwig Dec 12, 2023
1ba6ed9
Vertical Whitespace Violation
tom-ludwig Dec 12, 2023
13c49f2
Merge branch 'CodeEditApp:main' into search-speedUP-2.0
tom-ludwig Dec 16, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions CodeEdit/Features/Documents/Indexer/SearchIndexer+InternalMethods.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,13 +73,13 @@ extension SearchIndexer {
if isLeaf, inParentDocument != nil,
kSKDocumentStateNotIndexed != SKIndexGetDocumentState(index, inParentDocument) {
if let temp = SKDocumentCopyURL(inParentDocument) {
let baseURL = temp.takeUnretainedValue()
let baseURL = temp.takeUnretainedValue() as URL
let documentID = SKIndexGetDocumentID(index, inParentDocument)
docs.append(
DocumentID(
url: temp.takeRetainedValue() as URL,
url: baseURL,
docuemnt: inParentDocument!,
documentID: SKIndexGetDocumentID(index, inParentDocument)
documentID: documentID
)
)
}
Expand Down
273 changes: 174 additions & 99 deletions CodeEdit/Features/Documents/WorkspaceDocument+Search.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ extension WorkspaceDocument {

Task.detached {
let filePaths = self.getFileURLs(at: url)

let asyncController = SearchIndexer.AsyncManager(index: indexer)
var lastProgress: Double = 0

Expand Down Expand Up @@ -89,6 +90,23 @@ extension WorkspaceDocument {
return enumerator?.allObjects as? [URL] ?? []
}

/// Retrieves the contents of a files from the specified file paths.
///
/// - Parameter filePaths: An array of file URLs representing the paths of the files.
///
/// - Returns: An array of `TextFile` objects containing the standardized file URLs and text content.
func getfileContent(from filePaths: [URL]) async -> [SearchIndexer.AsyncManager.TextFile] {
var textFiles = [SearchIndexer.AsyncManager.TextFile]()
for file in filePaths {
if let content = try? String(contentsOf: file) {
textFiles.append(
SearchIndexer.AsyncManager.TextFile(url: file.standardizedFileURL, text: content)
)
}
}
return textFiles
}

/// Creates a search term based on the given query and search mode.
///
/// - Parameter query: The original user query string.
Expand Down Expand Up @@ -117,7 +135,7 @@ extension WorkspaceDocument {
/// within each file for the given string.
///
/// This method will update
/// ``WorkspaceDocument/SearchState-swift.class/searchResult``,
/// ``WorkspaceDocument/SearchState-swift.class/searchResult``,
/// ``WorkspaceDocument/SearchState-swift.class/searchResultsFileCount``
/// and ``WorkspaceDocument/SearchState-swift.class/searchResultCount`` with any matched
/// search results. See ``SearchResultModel`` and ``SearchResultMatchModel``
Expand All @@ -138,16 +156,16 @@ extension WorkspaceDocument {

let searchStream = await asyncController.search(query: searchQuery, 20)
for try await result in searchStream {
let urls2: [(URL, Float)] = result.results.map {
let urls: [(URL, Float)] = result.results.map {
($0.url, $0.score)
}

for (url, score) in urls2 {
for (url, score) in urls {
evaluateSearchQueue.async(group: evaluateResultGroup) {
evaluateResultGroup.enter()
Task {
var newResult = SearchResultModel(file: CEWorkspaceFile(url: url), score: score)
await self.evaluateResult(query: query.lowercased(), searchResult: &newResult)
await self.evaluateFile(query: query.lowercased(), searchResult: &newResult)

// Check if the new result has any line matches.
if !newResult.lineMatches.isEmpty {
Expand Down Expand Up @@ -190,115 +208,172 @@ extension WorkspaceDocument {
}
}

/// Adds line matchings to a `SearchResultsViewModel` array.
/// That means if a search result is a file, and the search term appears in the file,
/// the function will add the line number, line content, and keyword range to the `SearchResultsViewModel`.
/// Evaluates a search query within the content of a file and updates
/// the provided `SearchResultModel` with matching occurrences.
///
/// - Parameters:
/// - query: The search query string.
/// - searchResults: An inout parameter containing the array of `SearchResultsViewModel` to be evaluated.
/// It will be modified to include line matches.
private func evaluateResult(query: String, searchResult: inout SearchResultModel) async {
let searchResultCopy = searchResult
var newMatches = [SearchResultMatchModel]()

/// - query: The search query to be evaluated, potentially containing a regular expression.
/// - searchResult: The `SearchResultModel` object to be updated with the matching occurrences.
///
/// This function retrieves the content of a file specified in the `searchResult` parameter
/// and applies a search query using a regular expression.
/// It then iterates over the matches found in the file content,
/// creating `SearchResultMatchModel` instances for each match.
/// The resulting matches are appended to the `lineMatches` property of the `searchResult`.
/// Line matches are the preview lines that are shown in the search results.
///
/// # Example Usage
/// ```swift
/// var resultModel = SearchResultModel()
/// await evaluateFile(query: "example", searchResult: &resultModel)
/// ```
private func evaluateFile(query: String, searchResult: inout SearchResultModel) async {
guard let data = try? Data(contentsOf: searchResult.file.url),
let string = String(data: data, encoding: .utf8) else {
let fileContent = String(data: data, encoding: .utf8) else {
return
}

await withTaskGroup(of: SearchResultMatchModel?.self) { group in
for (lineNumber, line) in string.components(separatedBy: .newlines).lazy.enumerated() {
group.addTask {
let rawNoSpaceLine = line.trimmingCharacters(in: .whitespacesAndNewlines)
let noSpaceLine = rawNoSpaceLine.lowercased()
if self.lineContainsSearchTerm(line: noSpaceLine, term: query) {
let matches = noSpaceLine.ranges(of: query).map { range in
return [lineNumber, rawNoSpaceLine, range]
}
// Attempt to create a regular expression from the provided query
guard let regex = try? NSRegularExpression(pattern: query, options: [.caseInsensitive]) else {
return
}

for match in matches {
if let lineNumber = match[0] as? Int,
let lineContent = match[1] as? String,
let keywordRange = match[2] as? Range<String.Index> {
let matchModel = SearchResultMatchModel(
lineNumber: lineNumber,
file: searchResultCopy.file,
lineContent: lineContent,
keywordRange: keywordRange
)

return matchModel
}
}
}
return nil
}
for await groupRes in group {
if let groupRes {
newMatches.append(groupRes)
}
}
// Find all matches of the query within the file content using the regular expression
let matches = regex.matches(in: fileContent, range: NSRange(location: 0, length: fileContent.utf16.count))

var newMatches = [SearchResultMatchModel]()

// Process each match and add it to the array of `newMatches`
for match in matches {
if let matchRange = Range(match.range, in: fileContent) {
let matchWordLength = match.range.length
let matchModel = createMatchModel(
from: matchRange,
fileContent: fileContent,
file: searchResult.file,
matchWordLength: matchWordLength
)
newMatches.append(matchModel)
}
}

searchResult.lineMatches = newMatches
}

// see if the line contains search term, obeying selectedMode
// swiftlint:disable:next cyclomatic_complexity
func lineContainsSearchTerm(line rawLine: String, term searchterm: String) -> Bool {
var line = rawLine
if line.hasSuffix(" ") { line.removeLast() }
if line.hasPrefix(" ") { line.removeFirst() }

// Text
let findMode = selectedMode[1]
if findMode == .Text {
let textMatching = selectedMode[2]
let textContainsSearchTerm = line.contains(searchterm)
guard textContainsSearchTerm == true else { return false }
guard textMatching != .Containing else { return textContainsSearchTerm }

// get the index of the search term's appearance in the line
// and get the characters to the left and right
let appearances = line.appearancesOfSubstring(substring: searchterm, toLeft: 1, toRight: 1)
var foundMatch = false
for appearance in appearances {
let appearanceString = String(line[appearance])
guard appearanceString.count >= 2 else { continue }

var startsWith = false
var endsWith = false
if appearanceString.hasPrefix(searchterm) ||
!appearanceString.first!.isLetter ||
!(appearanceString.character(at: 2)?.isLetter ?? false) {
startsWith = true
}
if appearanceString.hasSuffix(searchterm) ||
!appearanceString.last!.isLetter ||
!(appearanceString.character(at: appearanceString.count-2)?.isLetter ?? false) {
endsWith = true
}
/// Creates a `SearchResultMatchModel` instance based on the provided parameters,
/// representing a matching occurrence within a file.
///
/// - Parameters:
/// - matchRange: The range of the matched substring within the entire file content.
/// - fileContent: The content of the file where the match was found.
/// - file: The `CEWorkspaceFile` object representing the file containing the match.
/// - matchWordLength: The length of the matched substring.
///
/// - Returns: A `SearchResultMatchModel` instance representing the matching occurrence.
///
/// This function is responsible for constructing a `SearchResultMatchModel`
/// based on the provided parameters. It extracts the relevant portions of the file content,
/// including the lines before and after the match, and combines them into a final line.
/// The resulting model includes information about the match's range within the file,
/// the file itself, the content of the line containing the match,
/// and the range of the matched keyword within that line.
private func createMatchModel(
from matchRange: Range<String.Index>,
fileContent: String,
file: CEWorkspaceFile,
matchWordLength: Int
) -> SearchResultMatchModel {
let preLine = extractPreLine(from: matchRange, fileContent: fileContent)
let keywordRange = extractKeywordRange(from: preLine, matchWordLength: matchWordLength)
let postLine = extractPostLine(from: matchRange, fileContent: fileContent)

let finalLine = preLine + postLine

return SearchResultMatchModel(
rangeWithinFile: matchRange,
file: file,
lineContent: finalLine,
keywordRange: keywordRange
)
}

switch textMatching {
case .MatchingWord:
foundMatch = startsWith && endsWith ? true : foundMatch
case .StartingWith:
foundMatch = startsWith ? true : foundMatch
case .EndingWith:
foundMatch = endsWith ? true : foundMatch
default: continue
}
}
return foundMatch
} else if findMode == .RegularExpression {
guard let regex = try? NSRegularExpression(pattern: searchterm) else { return false }
// swiftlint:disable:next legacy_constructor
return regex.firstMatch(in: String(line), range: NSMakeRange(0, line.utf16.count)) != nil
}
/// Extracts the line preceding a matching occurrence within a file.
///
/// - Parameters:
/// - matchRange: The range of the matched substring within the entire file content.
/// - fileContent: The content of the file where the match was found.
///
/// - Returns: A string representing the line preceding the match.
///
/// This function retrieves the line preceding a matching occurrence within the provided file content.
/// It considers a context of up to 60 characters before the match and clips the result to the last
/// occurrence of a newline character, ensuring that only the line containing the search term is displayed.
/// The extracted line is then trimmed of leading and trailing whitespaces and
/// newline characters before being returned.
private func extractPreLine(from matchRange: Range<String.Index>, fileContent: String) -> String {
let preRangeStart = fileContent.index(
matchRange.lowerBound,
offsetBy: -60,
limitedBy: fileContent.startIndex
) ?? fileContent.startIndex

let preRangeEnd = matchRange.upperBound
let preRange = preRangeStart..<preRangeEnd

let preLineWithNewLines = fileContent[preRange]
// Clip the range of the preview to the last occurrence of a new line
let lastNewLineIndexInPreLine = preLineWithNewLines.lastIndex(of: "\n") ?? preLineWithNewLines.startIndex
let preLineWithNewLinesPrefix = preLineWithNewLines[lastNewLineIndexInPreLine...]
return preLineWithNewLinesPrefix.trimmingCharacters(in: .whitespacesAndNewlines)
}

/// Extracts the range of the search term within the line preceding a matching occurrence.
///
/// - Parameters:
/// - preLine: The line preceding the matching occurrence within the file content.
/// - matchWordLength: The length of the search term.
///
/// - Returns: A range representing the position of the search term within the `preLine`.
///
/// This function calculates the range of the search term within
/// the provided line preceding a matching occurrence.
/// It considers the length of the search term to determine
/// the lower and upper bounds of the keyword range within the line.
private func extractKeywordRange(from preLine: String, matchWordLength: Int) -> Range<String.Index> {
let keywordLowerbound = preLine.index(
preLine.endIndex,
offsetBy: -matchWordLength,
limitedBy: preLine.startIndex
) ?? preLine.endIndex
let keywordUpperbound = preLine.endIndex
return keywordLowerbound..<keywordUpperbound
}

return false
// TODO: references and definitions
/// Extracts the line following a matching occurrence within a file.
///
/// - Parameters:
/// - matchRange: The range of the matched substring within the entire file content.
/// - fileContent: The content of the file where the match was found.
///
/// - Returns: A string representing the line following the match.
///
/// This function retrieves the line following a matching occurrence within the provided file content.
/// It considers a context of up to 60 characters after the match and clips the result to the first
/// occurrence of a newline character, ensuring that only the relevant portion of the line is displayed.
/// The extracted line is then converted to a string before being returned.
private func extractPostLine(from matchRange: Range<String.Index>, fileContent: String) -> String {
let postRangeStart = matchRange.upperBound
let postRangeEnd = fileContent.index(
matchRange.upperBound,
offsetBy: 60,
limitedBy: fileContent.endIndex
) ?? fileContent.endIndex

let postRange = postRangeStart..<postRangeEnd
let postLineWithNewLines = fileContent[postRange]

let firstNewLineIndexInPostLine = postLineWithNewLines.firstIndex(of: "\n") ?? postLineWithNewLines.endIndex
return String(postLineWithNewLines[..<firstNewLineIndexInPostLine])
}

/// Resets the search results along with counts for overall results and file-specific results.
Expand Down
Loading