[WIP] Link to external docsets (mainly apple) when parsing a usr in a declaration. Additionally, creates a "link" to a usr for any type, function, etc. mentioned in a doc comment.

Package.resolved

@@ -15,17 +15,26 @@
         "repositoryURL": "https://github.com/Carthage/Commandant.git",
         "state": {
           "branch": null,
-          "revision": "7f29606ec3a2054a601f0e72f562a104dbc1a11a",
-          "version": "0.13.0"
+          "revision": "066bf9a79c37cf24fe4746ab7a1793b0359d1fe5",
+          "version": "0.14.0"
+        }
+      },
+      {
+        "package": "CSQLite",
+        "repositoryURL": "https://github.com/groue/CSQLite.git",
+        "state": {
+          "branch": null,
+          "revision": "51210b121508dd91dcced13d398269c004b0f1b5",
+          "version": "0.2.0"
         }
       },
       {
         "package": "Nimble",
         "repositoryURL": "https://github.com/Quick/Nimble.git",
         "state": {
           "branch": null,
-          "revision": "21f4fed2052cea480f5f1d2044d45aa25fdfb988",
-          "version": "7.1.1"
+          "revision": "8023e3980d91b470ad073d6da843b73f2eeb1844",
+          "version": "7.1.2"
         }
       },
       {

Package.swift

@@ -27,12 +27,17 @@ let package = Package(
             dependencies: [
                 "SWXMLHash",
                 "Yams",
+                "CSQLite"
             ],
             exclude: [
                 "clang-c",
                 "sourcekitd.h",
             ]
         ),
+        .target(
+            name: "CSQLite",
+            dependencies: []
+        ),
         .testTarget(
             name: "SourceKittenFrameworkTests",
             dependencies: [

Source/CSQLite/anchor.c

@@ -0,0 +1 @@
+/* Empty file so SPM will build this target */

Source/CSQLite/include/module.modulemap

@@ -0,0 +1,5 @@
+module CSQLite [system] {
+    header "shim.h"
+    link "sqlite3"
+    export *
+}

Source/CSQLite/include/shim.h

@@ -0,0 +1,14 @@
+#ifndef CSQLITE_SHIM_H
+#define CSQLITE_SHIM_H
+
+#include <sqlite3.h>
+
+typedef void(*errorLogCallback)(void *pArg, int iErrCode, const char *zMsg);
+
+// Wrapper around sqlite3_config(SQLITE_CONFIG_LOG, ...) which is a variadic
+// function that can't be used from Swift.
+static inline void registerErrorLogCallback(errorLogCallback callback) {
+    sqlite3_config(SQLITE_CONFIG_LOG, callback, 0);
+}
+
+#endif // CSQLITE_SHIM_H

Source/SourceKittenFramework/Clang+SourceKitten.swift

@@ -32,7 +32,7 @@ private func setUUIDString(uidString: String, `for` file: String) {
 }
 
 struct ClangIndex {
-    private let index = clang_createIndex(0, 1)
+    public let index = clang_createIndex(0, 1)
 
     func open(file: String, args: [UnsafePointer<Int8>?]) -> CXTranslationUnit {
         return clang_createTranslationUnitFromSourceFile(index, file, Int32(args.count), args, 0, nil)!
@@ -71,10 +71,7 @@ extension CXCursor {
     }
 
     func extent() -> (start: SourceLocation, end: SourceLocation) {
-        let extent = clang_getCursorExtent(self)
-        let start = SourceLocation(clangLocation: clang_getRangeStart(extent))
-        let end = SourceLocation(clangLocation: clang_getRangeEnd(extent))
-        return (start, end)
+        return clang_getCursorExtent(self).range()
     }
 
     func shouldDocument() -> Bool {
@@ -193,6 +190,30 @@ extension CXCursor {
         }
         return commentBody
     }
+
+    func commentBodyExtent() -> (start: SourceLocation, end: SourceLocation) {
+        return clang_Cursor_getCommentRange(self).range()
+    }
+
+    func tokensAndCursors(handle: (([CXToken], [CXCursor]) -> Void)) {
+        guard let translation = clang_Cursor_getTranslationUnit(self) else { return }
+        var token : CXToken = CXToken.init()
+        var numTokens : UInt32 = 0
+        withUnsafeMutablePointer(to: &token) { (pointer) in
+            var p = Optional(pointer)
+            clang_tokenize(translation, clang_getCursorExtent(self), &p, &numTokens)
+            var tokens = Array(UnsafeBufferPointer.init(start: p, count: Int(numTokens)))
+            var cursors = Array.init(repeating: CXCursor.init(), count: Int(numTokens))
+            //let cursorPointer = malloc(MemoryLayout.size(ofValue: CXCursor.self)*Int(numTokens)).assumingMemoryBound(to: CXCursor.self)
+            //var realPointer = UnsafeMutablePointer.init(mutating: (tokens.withUnsafeBufferPointer {return $0}).baseAddress)
+            clang_annotateTokens(translation, &tokens, numTokens, &cursors)
+            //let cursors = Array(UnsafeBufferPointer.init(start: cursorPointer, count: Int(numTokens)))
+
+            handle(tokens, cursors)
+
+            clang_disposeTokens(translation, p, numTokens)
+        }
+    }
 
     func swiftDeclarationAndName(compilerArguments: [String]) -> (swiftDeclaration: String?, swiftName: String?) {
         let file = location().file
@@ -286,4 +307,13 @@ extension CXComment {
     }
 }
 
+extension CXSourceRange {
+    func range() -> (start: SourceLocation, end: SourceLocation) {
+        let start = SourceLocation(clangLocation: clang_getRangeStart(self))
+        let end = SourceLocation(clangLocation: clang_getRangeEnd(self))
+
+        return (start: start, end: end)
+    }
+}
+
 #endif

Source/SourceKittenFramework/ClangTranslationUnit.swift

@@ -61,7 +61,7 @@ public struct ClangTranslationUnit {
         let clangIndex = ClangIndex()
         clangTranslationUnits = headerFiles.map { clangIndex.open(file: $0, args: cStringCompilerArguments) }
         declarations = clangTranslationUnits
-            .flatMap { $0.cursor().compactMap({ SourceDeclaration(cursor: $0, compilerArguments: compilerArguments) }) }
+            .flatMap { $0.cursor().compactMap({ SourceDeclaration(cursor: $0, compilerArguments: compilerArguments, index: clangIndex.index) }) }
             .rejectEmptyDuplicateEnums()
             .distinct()
             .sorted()

Source/SourceKittenFramework/File.swift

@@ -149,6 +149,50 @@ public final class File {
         return substring?.removingCommonLeadingWhitespaceFromLines()
                          .trimmingWhitespaceAndOpeningCurlyBrace()
     }
+
+    /**
+    Parse the annotated declaration for any usr links. If they are external (i.e. apple ones) include a link to the external documentation.
+
+     - parameter dictionary: SourceKit dictionary to extract declaration from.
+
+     - returns: Source declaration if successfully parsed, with any usr links. Example: `public class Test : <USRLINK usr="s:SS" url="https://developer.apple.com/documentation/swift/string">`.
+    */
+    public func parseAnnotatedDeclaration(_ dictionary: [String: SourceKitRepresentable]) -> String? {
+        guard let annotated = SwiftDocKey.getAnnotatedDeclaration(dictionary) else {
+            return nil
+        }
+
+        guard let decl = SWXMLHash.parse(annotated).children.first?.element else {
+            return nil
+        }
+
+        return recurseAnnotatedXML(decl)
+    }
+
+    private func recurseAnnotatedXML(_ element: SWXMLHash.XMLElement) -> String {
+        let parsed = element.children.map({ (child) -> String in
+            if let text = child as? TextElement {
+                return text.text
+            }
+
+            if let xml = child as? SWXMLHash.XMLElement {
+                let content = recurseAnnotatedXML(xml)
+                if let usr = xml.attribute(by: "usr") {
+                    if let url = USRResolver.shared.resolveExternalURL(usr: usr.text) {
+                        return "<USRLINK usr=\"\(usr.text)\" url=\"\(url)\">\(content)</USRLINK>"
+                    } else {
+                        return "<USRLINK usr=\"\(usr.text)\">\(content)</USRLINK>"
+                    }
+                } else {
+                    return content
+                }
+            }
+
+            return child.description
+        })
+
+        return parsed.joined(separator: "")
+    }
 
     /**
     Parse line numbers containing the declaration's implementation from SourceKit dictionary.
@@ -210,6 +254,11 @@ public final class File {
         if let parsedDeclaration = parseDeclaration(dictionary) {
             dictionary[SwiftDocKey.parsedDeclaration.rawValue] = parsedDeclaration
         }
+
+        // Parse annotated declaration and add to dictionary
+        if let parsedAnnotatedDeclaration = parseAnnotatedDeclaration(dictionary) {
+            dictionary[SwiftDocKey.parsedAnnotatedDeclaration.rawValue] = parsedAnnotatedDeclaration
+        }
 
         // Parse scope range and add to dictionary
         if let parsedScopeRange = parseScopeRange(dictionary) {
@@ -414,7 +463,6 @@ public final class File {
      */
     internal func addDocComments(dictionary: [String: SourceKitRepresentable], finder: SyntaxMap.DocCommentFinder) -> [String: SourceKitRepresentable] {
         var dictionary = dictionary
-
         // special-case skip 'enumcase': has same offset as child 'enumelement'
         if let kind = SwiftDocKey.getKind(dictionary).flatMap(SwiftDeclarationKind.init),
            kind != .enumcase,
@@ -435,6 +483,35 @@ public final class File {
 
         return dictionary
     }
+
+    internal func parseDocComments(dictionary: [String: SourceKitRepresentable], parentUSR: String? = nil) -> [String: SourceKitRepresentable] {
+        var dictionary = dictionary
+        let currentUSR = dictionary[SwiftDocKey.usr.rawValue] as? String ?? ""
+        let kind = SwiftDocKey.getKind(dictionary)
+        if let docComment = dictionary[SwiftDocKey.documentationComment.rawValue] as? String {
+            var result = docComment
+            var start = result.startIndex
+            while var range = result.range(of: "`.*?`", options: .regularExpression, range: start..<result.endIndex) {
+                let code = String(result[range]).replacingOccurrences(of: "`", with: "")
+                let context = NameEntity(usr: currentUSR, name: "context", children: [], parentUSR: parentUSR, kind: SwiftDeclarationKind(rawValue: kind ?? "") ?? SwiftDeclarationKind.class)
+                if let usr = USRResolver.shared.resolveUSR(code: code, context: context) {
+                    let replacement = "`<USRLINK usr=\"\(usr)\">\(code)</USRLINK>`"
+                    result = result.replacingCharacters(in: range, with: replacement)
+                    range = range.lowerBound..<result.index(range.lowerBound, offsetBy: replacement.count)
+                }
+                start = range.lowerBound < range.upperBound ? range.upperBound : result.index(range.lowerBound, offsetBy: 1, limitedBy: result.endIndex) ?? result.endIndex
+            }
+            dictionary[SwiftDocKey.parsedDocumentationComment.rawValue] = result
+        }
+
+        if let substructure = SwiftDocKey.getSubstructure(dictionary) {
+            dictionary[SwiftDocKey.substructure.rawValue] = substructure.map {
+                parseDocComments(dictionary: $0, parentUSR: currentUSR)
+            }
+        }
+
+        return dictionary
+    }
 }
 
 /**

Source/SourceKittenFramework/JSONOutput.swift

@@ -99,6 +99,7 @@ private func toOutputDictionary(_ decl: SourceDeclaration) -> [String: Any] {
     set(.name, decl.name)
     set(.usr, decl.usr)
     set(.parsedDeclaration, decl.declaration)
+    set(.parsedAnnotatedDeclaration, decl.annotatedDeclaration)
     set(.documentationComment, decl.commentBody)
     set(.parsedScopeStart, Int(decl.extent.start.line))
     set(.parsedScopeEnd, Int(decl.extent.end.line))

Source/SourceKittenFramework/Module.swift

@@ -22,7 +22,8 @@ public struct Module {
     public var docs: [SwiftDocs] {
         var fileIndex = 1
         let sourceFilesCount = sourceFiles.count
-        return sourceFiles.sorted().compactMap {
+        // HACKY AF CURRENTLY!! NEED TO ADD THIS SOMEWHERE ELSE; BUT WHERE?
+        var d : [SwiftDocs] = sourceFiles.sorted().compactMap {
             let filename = $0.bridge().lastPathComponent
             if let file = File(path: $0) {
                 fputs("Parsing \(filename) (\(fileIndex)/\(sourceFilesCount))\n", stderr)
@@ -32,6 +33,19 @@ public struct Module {
             fputs("Could not parse `\(filename)`. Please open an issue at https://github.com/jpsim/SourceKitten/issues with the file contents.\n", stderr)
             return nil
         }
+
+        var d2 : [SwiftDocs] = []
+
+        for var sd in d {
+            USRResolver.shared.register(docs: sd.docsDictionary)
+        }
+
+        for var sd in d {
+            sd.parseDocComments()
+            d2.append(sd)
+        }
+
+        return d2
     }
 
     public init?(spmName: String) {