Add support for parsing with rust-analyzer instead of librustc_parse

This adds a new librustc_parse_ra library that implements the same interface of librustc_parse, but uses rust-analyzer to parse the code, and then converts the rust-analyzer AST to the rustc AST, and adds a new -Z parse-with-rust-analyzer option that makes rustc use it, gated by a new rust_analyzer feature flag and config.toml option. The code is preliminary but should be essentially complete, and capable of building large projects that only use the Rust 2018 edition (Rust 2015 is not currently supported by rust-analyzer). Note that this is likely to accept invalid code, is not fully complete, and provides worse error messages than the current parser, so it cannot be stabilized without significant additional work.
rust-lang · Apr 4, 2020 · 3f40227 · 3f40227
1 parent 1b521f5
commit 3f40227
Show file tree

Hide file tree

Showing 13 changed files with 3,261 additions and 11 deletions.
diff --git a/Cargo.lock b/Cargo.lock
@@ -740,7 +740,7 @@ dependencies = [
  "crossbeam-utils 0.6.5",
  "lazy_static 1.4.0",
  "memoffset",
- "scopeguard",
+ "scopeguard 1.0.0",
 ]
 
 [[package]]
@@ -952,6 +952,12 @@ dependencies = [
  "rustc-std-workspace-core",
 ]
 
+[[package]]
+name = "drop_bomb"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "69b26e475fd29098530e709294e94e661974c851aed42512793f120fed4e199f"
+
 [[package]]
 name = "dtoa"
 version = "0.4.4"
@@ -1404,7 +1410,7 @@ version = "0.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a3753954f7bd71f0e671afb8b5a992d1724cf43b7f95a563cd4a0bde94659ca8"
 dependencies = [
- "scopeguard",
+ "scopeguard 1.0.0",
  "winapi 0.3.8",
 ]
 
@@ -1864,13 +1870,23 @@ version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ae91b68aebc4ddb91978b11a1b02ddd8602a05ec19002801c5666000e05e0f83"
 
+[[package]]
+name = "lock_api"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "62ebf1391f6acad60e5c8b43706dde4582df75c06698ab44511d15016bc2442c"
+dependencies = [
+ "owning_ref",
+ "scopeguard 0.3.3",
+]
+
 [[package]]
 name = "lock_api"
 version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f8912e782533a93a167888781b836336a6ca5da6175c05944c86cf28c31104dc"
 dependencies = [
- "scopeguard",
+ "scopeguard 1.0.0",
 ]
 
 [[package]]
@@ -2345,6 +2361,15 @@ dependencies = [
  "winapi 0.3.8",
 ]
 
+[[package]]
+name = "owning_ref"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ff55baddef9e4ad00f88b6c743a2a8062d4c6ade126c2a528644b8e444d52ce"
+dependencies = [
+ "stable_deref_trait",
+]
+
 [[package]]
 name = "packed_simd"
 version = "0.3.1"
@@ -2393,13 +2418,33 @@ dependencies = [
  "winapi 0.3.8",
 ]
 
+[[package]]
+name = "parking_lot"
+version = "0.6.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f0802bff09003b291ba756dc7e79313e51cc31667e94afbe847def490424cde5"
+dependencies = [
+ "lock_api 0.1.5",
+ "parking_lot_core 0.3.1",
+]
+
+[[package]]
+name = "parking_lot"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ab41b4aed082705d1056416ae4468b6ea99d52599ecf3169b00088d43113e337"
+dependencies = [
+ "lock_api 0.1.5",
+ "parking_lot_core 0.4.0",
+]
+
 [[package]]
 name = "parking_lot"
 version = "0.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f842b1982eb6c2fe34036a4fbfb06dd185a3f5c8edfaacdf7d1ea10b07de6252"
 dependencies = [
- "lock_api",
+ "lock_api 0.3.1",
  "parking_lot_core 0.6.2",
  "rustc_version",
 ]
@@ -2410,10 +2455,36 @@ version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "92e98c49ab0b7ce5b222f2cc9193fc4efe11c6d0bd4f648e374684a6857b1cfc"
 dependencies = [
- "lock_api",
+ "lock_api 0.3.1",
  "parking_lot_core 0.7.0",
 ]
 
+[[package]]
+name = "parking_lot_core"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ad7f7e6ebdc79edff6fdcb87a55b620174f7a989e3eb31b65231f4af57f00b8c"
+dependencies = [
+ "libc",
+ "rand 0.5.6",
+ "rustc_version",
+ "smallvec 0.6.10",
+ "winapi 0.3.8",
+]
+
+[[package]]
+name = "parking_lot_core"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "94c8c7923936b28d546dfd14d4472eaf34c99b14e1c973a32b3e6d4eb04298c9"
+dependencies = [
+ "libc",
+ "rand 0.6.1",
+ "rustc_version",
+ "smallvec 0.6.10",
+ "winapi 0.3.8",
+]
+
 [[package]]
 name = "parking_lot_core"
 version = "0.6.2"
@@ -2713,6 +2784,20 @@ dependencies = [
  "proc-macro2 1.0.3",
 ]
 
+[[package]]
+name = "ra_syntax"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "be308e1907623f15bc874a4b2742b6a20c44c439f201638fa26564e664276ce9"
+dependencies = [
+ "drop_bomb",
+ "itertools 0.7.8",
+ "parking_lot 0.6.4",
+ "rowan",
+ "text_unit",
+ "unicode-xid 0.1.0",
+]
+
 [[package]]
 name = "racer"
 version = "2.1.31"
@@ -2736,6 +2821,19 @@ dependencies = [
  "rustc-ap-syntax",
 ]
 
+[[package]]
+name = "rand"
+version = "0.5.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c618c47cd3ebd209790115ab837de41425723956ad3ce2e6a7f09890947cacb9"
+dependencies = [
+ "cloudabi",
+ "fuchsia-cprng",
+ "libc",
+ "rand_core 0.3.0",
+ "winapi 0.3.8",
+]
+
 [[package]]
 name = "rand"
 version = "0.6.1"
@@ -3141,6 +3239,17 @@ dependencies = [
  "rls-span",
 ]
 
+[[package]]
+name = "rowan"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c218b4430ab922850b71b14fa9bca224425097f935f6155c0b6a4b1f398a54f0"
+dependencies = [
+ "parking_lot 0.7.1",
+ "smol_str",
+ "text_unit",
+]
+
 [[package]]
 name = "rustbook"
 version = "0.1.0"
@@ -3849,6 +3958,7 @@ dependencies = [
  "rustc_mir",
  "rustc_mir_build",
  "rustc_parse",
+ "rustc_parse_ra",
  "rustc_passes",
  "rustc_plugin_impl",
  "rustc_privacy",
@@ -4045,6 +4155,24 @@ dependencies = [
  "unicode-normalization",
 ]
 
+[[package]]
+name = "rustc_parse_ra"
+version = "0.0.0"
+dependencies = [
+ "bitflags",
+ "log",
+ "ra_syntax",
+ "rustc_ast",
+ "rustc_ast_pretty",
+ "rustc_data_structures",
+ "rustc_errors",
+ "rustc_parse",
+ "rustc_session",
+ "rustc_span",
+ "smallvec 1.0.0",
+ "unicode-normalization",
+]
+
 [[package]]
 name = "rustc_passes"
 version = "0.0.0"
@@ -4443,6 +4571,12 @@ version = "0.1.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1d51f5df5af43ab3f1360b429fa5e0152ac5ce8c0bd6485cae490332e96846a8"
 
+[[package]]
+name = "scopeguard"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "94258f53601af11e6a49f722422f6e3425c52b06245a5cf9bc09908b174f5e27"
+
 [[package]]
 name = "scopeguard"
 version = "1.0.0"
@@ -4625,6 +4759,12 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4ecf3b85f68e8abaa7555aa5abdb1153079387e60b718283d732f03897fcfc86"
 
+[[package]]
+name = "smol_str"
+version = "0.1.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "34836c9a295c62c2ce3514471117c5cb269891e8421b2aafdd910050576c4d8b"
+
 [[package]]
 name = "socket2"
 version = "0.3.11"
@@ -4921,6 +5061,12 @@ dependencies = [
  "term 0.6.0",
 ]
 
+[[package]]
+name = "text_unit"
+version = "0.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "20431e104bfecc1a40872578dbc390e10290a0e9c35fffe3ce6f73c15a9dbfc2"
+
 [[package]]
 name = "textwrap"
 version = "0.11.0"

diff --git a/config.toml.example b/config.toml.example
@@ -436,6 +436,9 @@
 # override the default allocator for rustc and LLVM.
 #jemalloc = false
 
+# Enable experimental features that depend on rust-analyzer.
+#rust-analyzer = false
+
 # Run tests in various test suites with the "nll compare mode" in addition to
 # running the tests in normal mode. Largely only used on CI and during local
 # development of NLL

diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs
@@ -117,6 +117,7 @@ pub struct Config {
     pub targets: Vec<Interned<String>>,
     pub local_rebuild: bool,
     pub jemalloc: bool,
+    pub rust_analyzer: bool,
     pub control_flow_guard: bool,
 
     // dist misc
@@ -342,6 +343,7 @@ struct Rust {
     thin_lto_import_instr_limit: Option<u32>,
     remap_debuginfo: Option<bool>,
     jemalloc: Option<bool>,
+    rust_analyzer: Option<bool>,
     test_compare_mode: Option<bool>,
     llvm_libunwind: Option<bool>,
     control_flow_guard: Option<bool>,
@@ -571,6 +573,7 @@ impl Config {
             set(&mut config.codegen_tests, rust.codegen_tests);
             set(&mut config.rust_rpath, rust.rpath);
             set(&mut config.jemalloc, rust.jemalloc);
+            set(&mut config.rust_analyzer, rust.rust_analyzer);
             set(&mut config.test_compare_mode, rust.test_compare_mode);
             set(&mut config.llvm_libunwind, rust.llvm_libunwind);
             set(&mut config.backtrace, rust.backtrace);

diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs
@@ -509,6 +509,9 @@ impl Build {
         if self.config.jemalloc {
             features.push_str("jemalloc");
         }
+        if self.config.rust_analyzer {
+            features.push_str("rust_analyzer");
+        }
         if self.config.llvm_enabled() {
             features.push_str(" llvm");
         }

diff --git a/src/librustc_driver/Cargo.toml b/src/librustc_driver/Cargo.toml
@@ -40,3 +40,4 @@ winapi = { version = "0.3", features = ["consoleapi", "debugapi", "processenv"]
 
 [features]
 llvm = ['rustc_interface/llvm']
+rust_analyzer = ['rustc_interface/rust_analyzer']
diff --git a/src/librustc_interface/Cargo.toml b/src/librustc_interface/Cargo.toml
@@ -18,6 +18,7 @@ rustc_attr = { path = "../librustc_attr" }
 rustc_builtin_macros = { path = "../librustc_builtin_macros" }
 rustc_expand = { path = "../librustc_expand" }
 rustc_parse = { path = "../librustc_parse" }
+rustc_parse_ra = { path = "../librustc_parse_ra", optional = true }
 rustc_session = { path = "../librustc_session" }
 rustc_span = { path = "../librustc_span" }
 rustc_serialize = { path = "../libserialize", package = "serialize" }
@@ -55,3 +56,4 @@ rustc_target = { path = "../librustc_target" }
 
 [features]
 llvm = ['rustc_codegen_llvm']
+rust_analyzer = ['rustc_parse_ra']
diff --git a/src/librustc_interface/passes.rs b/src/librustc_interface/passes.rs
@@ -50,13 +50,40 @@ use std::path::PathBuf;
 use std::rc::Rc;
 use std::{env, fs, iter, mem};
 
-pub fn parse<'a>(sess: &'a Session, input: &Input) -> PResult<'a, ast::Crate> {
-    let krate = sess.time("parse_crate", || match input {
-        Input::File(file) => parse_crate_from_file(file, &sess.parse_sess),
+#[cfg(rust_analyzer)]
+fn parse_with_ra<'a>(sess: &'a Session, input: &Input) -> PResult<'a, ast::Crate>
+{
+    match input {
+        Input::File(file) => {
+            rustc_parse_ra::parse_crate_from_file(file, &sess.parse_sess)
+        },
         Input::Str { input, name } => {
-            parse_crate_from_source_str(name.clone(), input.clone(), &sess.parse_sess)
+            rustc_parse_ra::parse_crate_from_source_str(name.clone(), input.clone(), &sess.parse_sess)
         }
-    })?;
+    }
+}
+
+#[cfg(not(rust_analyzer))]
+fn parse_with_ra<'a>(sess: &'a Session, _input: &Input) -> PResult<'a, ast::Crate>
+{
+    Err(sess.struct_fatal("parsing with rust-analyzer is not supported because this version of rustc was not compiled with the \"rust_analyzer\" feature flag enabled"))
+}
+
+pub fn parse<'a>(sess: &'a Session, input: &Input) -> PResult<'a, ast::Crate> {
+    let krate = sess.time("parse_crate", ||
+        if sess.opts.debugging_opts.parse_with_rust_analyzer {
+            parse_with_ra(sess, input)
+        } else {
+            match input {
+                Input::File(file) => {
+                    parse_crate_from_file(file, &sess.parse_sess)
+                },
+                Input::Str { input, name } => {
+                    parse_crate_from_source_str(name.clone(), input.clone(), &sess.parse_sess)
+                }
+            }
+        }
+    )?;
 
     if sess.opts.debugging_opts.ast_json_noexpand {
         println!("{}", json::as_json(&krate));

diff --git a/src/librustc_parse/lexer/mod.rs b/src/librustc_parse/lexer/mod.rs
@@ -83,6 +83,7 @@ impl<'a> StringReader<'a> {
         let mut sr = StringReader::new(sess, begin.sf, None);
 
         // Seek the lexer to the right byte range.
+        sr.pos = span.lo();
         sr.end_src_index = sr.src_index(span.hi());
 
         sr