refactor: CompilationDatabase and scan

.gitignore

@@ -49,6 +49,9 @@ temp/
 __pycache__/
 .pytest_cache/
 tests/unit/Local/
+perf.data
+flamegraph.svg
+
 
 **/node_modules
 docs/.vitepress/dist

cmake/package.cmake

@@ -23,6 +23,7 @@ if(CMAKE_BUILD_TYPE STREQUAL "Debug")
         LLVMSupport
         LLVMFrontendOpenMP
         LLVMOption
+        LLVMTargetParser
         clangAST
         clangASTMatchers
         clangBasic

include/AST/SourceCode.h

@@ -33,6 +33,8 @@ clang::SourceLocation get(clang::SourceRange range) {
     }
 }
 
+class Lexer;
+
 }  // namespace clang
 
 namespace clice {
@@ -63,15 +65,17 @@ struct LocalSourceRange {
     }
 };
 
+using TokenKind = clang::tok::TokenKind;
+
 struct Token {
     /// Whether this token is at the start of line.
     bool is_at_start_of_line = false;
 
     /// Whether this token is a preprocessor directive.
-    bool is_preprocessor_directive = false;
+    bool is_pp_keyword = false;
 
     /// The kind of this token.
-    clang::tok::TokenKind kind;
+    TokenKind kind;
 
     /// The source range of this token.
     LocalSourceRange range;
@@ -80,31 +84,35 @@ struct Token {
         return range.valid();
     }
 
-    llvm::StringRef name() {
+    llvm::StringRef name() const {
         return clang::tok::getTokenName(kind);
     }
 
-    llvm::StringRef text(llvm::StringRef content) {
+    llvm::StringRef text(llvm::StringRef content) const {
         assert(range.valid() && "Invalid source range");
         return content.substr(range.begin, range.end - range.begin);
     }
 
-    bool is_eof() {
+    bool is_eod() const {
+        return kind == clang::tok::eod;
+    }
+
+    bool is_eof() const {
         return kind == clang::tok::eof;
     }
 
-    bool is_identifier() {
+    bool is_identifier() const {
         return kind == clang::tok::raw_identifier;
     }
 
-    bool is_directive_hash() {
+    bool is_directive_hash() const {
         return is_at_start_of_line && kind == clang::tok::hash;
     }
 
     /// The tokens after the include diretive are regarded as
     /// a whole token, whose kind is `header_name`. For example
     /// `<iostream>` and `"test.h"` are both header name.
-    bool is_header_name() {
+    bool is_header_name() const {
         return kind == clang::tok::header_name;
     }
 };
@@ -120,6 +128,10 @@ class Lexer {
 
     Lexer(Lexer&&) = delete;
 
+    Lexer& operator= (const Lexer&) = delete;
+
+    Lexer& operator= (Lexer&&) = delete;
+
     ~Lexer();
 
     void lex(Token& token);
@@ -133,20 +145,35 @@ class Lexer {
     /// Advance the lexer and return the next token.
     Token advance();
 
+    /// Advance the lexer if the next token kind is the param.
+    std::optional<Token> advance_if(llvm::function_ref<bool(const Token&)> callback);
+
+    std::optional<Token> advance_if(llvm::StringRef spelling) {
+        return advance_if([&](const Token& token) {
+            return token.is_identifier() && token.text(content) == spelling;
+        });
+    }
+
+    std::optional<Token> advance_if(TokenKind kind) {
+        return advance_if([&](const Token& token) { return token.kind == kind; });
+    }
+
     /// Advance the lexer until meet the specific kind token.
-    Token advance_until(clang::tok::TokenKind kind);
+    Token advance_until(TokenKind kind);
 
 private:
     /// If this is set to false, the lexer will emit tok::eod at the end
     /// of directive.
     bool ignore_end_of_directive = true;
 
     /// Whether we are lexing the preprocessor directive.
-    bool parse_preprocessor_directive = false;
+    bool parse_pp_keyword = false;
 
     /// Whether we are lexing the header name.
     bool parse_header_name = false;
 
+    bool module_declaration_context = true;
+
     /// The cache of last token.
     Token last_token;
 
@@ -159,7 +186,7 @@ class Lexer {
     /// The lexed content.
     llvm::StringRef content;
 
-    void* impl;
+    std::unique_ptr<clang::Lexer> lexer;
 };
 
 }  // namespace clice