Rollup merge of rust-lang#61026 - estebank:macro-eof-spans, r=petroch…

…enkov

Tweak macro parse errors when reaching EOF during macro call parse

Add detail on origin of current parser when reaching EOF, stop saying "found `<eof>`" and point at the end of macro calls.

Fix rust-lang#27569.

src/librustc/traits/on_unimplemented.rs

@@ -226,12 +226,12 @@ impl<'a, 'gcx, 'tcx> OnUnimplementedFormatString {
         Ok(result)
     }
 
-    fn verify(&self,
-              tcx: TyCtxt<'a, 'gcx, 'tcx>,
-              trait_def_id: DefId,
-              span: Span)
-              -> Result<(), ErrorReported>
-    {
+    fn verify(
+        &self,
+        tcx: TyCtxt<'a, 'gcx, 'tcx>,
+        trait_def_id: DefId,
+        span: Span,
+    ) -> Result<(), ErrorReported> {
         let name = tcx.item_name(trait_def_id);
         let generics = tcx.generics_of(trait_def_id);
         let parser = Parser::new(&self.0, None, vec![], false);
@@ -272,12 +272,12 @@ impl<'a, 'gcx, 'tcx> OnUnimplementedFormatString {
         result
     }
 
-    pub fn format(&self,
-                  tcx: TyCtxt<'a, 'gcx, 'tcx>,
-                  trait_ref: ty::TraitRef<'tcx>,
-                  options: &FxHashMap<String, String>)
-                  -> String
-    {
+    pub fn format(
+        &self,
+        tcx: TyCtxt<'a, 'gcx, 'tcx>,
+        trait_ref: ty::TraitRef<'tcx>,
+        options: &FxHashMap<String, String>,
+    ) -> String {
         let name = tcx.item_name(trait_ref.def_id);
         let trait_str = tcx.def_path_str(trait_ref.def_id);
         let generics = tcx.generics_of(trait_ref.def_id);

src/libsyntax/attr/mod.rs

@@ -278,7 +278,14 @@ impl Attribute {
     pub fn parse<'a, T, F>(&self, sess: &'a ParseSess, mut f: F) -> PResult<'a, T>
         where F: FnMut(&mut Parser<'a>) -> PResult<'a, T>,
     {
-        let mut parser = Parser::new(sess, self.tokens.clone(), None, false, false);
+        let mut parser = Parser::new(
+            sess,
+            self.tokens.clone(),
+            None,
+            false,
+            false,
+            Some("attribute"),
+        );
         let result = f(&mut parser)?;
         if parser.token != token::Eof {
             parser.unexpected()?;

src/libsyntax/ext/base.rs

@@ -11,7 +11,7 @@ use crate::parse::{self, parser, DirectoryOwnership};
 use crate::parse::token;
 use crate::ptr::P;
 use crate::symbol::{kw, sym, Ident, Symbol};
-use crate::ThinVec;
+use crate::{ThinVec, MACRO_ARGUMENTS};
 use crate::tokenstream::{self, TokenStream};
 
 use errors::{DiagnosticBuilder, DiagnosticId};
@@ -850,7 +850,7 @@ impl<'a> ExtCtxt<'a> {
     }
 
     pub fn new_parser_from_tts(&self, tts: &[tokenstream::TokenTree]) -> parser::Parser<'a> {
-        parse::stream_to_parser(self.parse_sess, tts.iter().cloned().collect())
+        parse::stream_to_parser(self.parse_sess, tts.iter().cloned().collect(), MACRO_ARGUMENTS)
     }
     pub fn source_map(&self) -> &'a SourceMap { self.parse_sess.source_map() }
     pub fn parse_sess(&self) -> &'a parse::ParseSess { self.parse_sess }

src/libsyntax/ext/tt/macro_parser.rs

@@ -658,7 +658,14 @@ pub fn parse(
     recurse_into_modules: bool,
 ) -> NamedParseResult {
     // Create a parser that can be used for the "black box" parts.
-    let mut parser = Parser::new(sess, tts, directory, recurse_into_modules, true);
+    let mut parser = Parser::new(
+        sess,
+        tts,
+        directory,
+        recurse_into_modules,
+        true,
+        crate::MACRO_ARGUMENTS,
+    );
 
     // A queue of possible matcher positions. We initialize it with the matcher position in which
     // the "dot" is before the first token of the first token tree in `ms`. `inner_parse_loop` then

src/libsyntax/ext/tt/macro_rules.rs

@@ -172,7 +172,7 @@ fn generic_extension<'cx>(cx: &'cx mut ExtCtxt<'_>,
                     path: Cow::from(cx.current_expansion.module.directory.as_path()),
                     ownership: cx.current_expansion.directory_ownership,
                 };
-                let mut p = Parser::new(cx.parse_sess(), tts, Some(directory), true, false);
+                let mut p = Parser::new(cx.parse_sess(), tts, Some(directory), true, false, None);
                 p.root_module_name = cx.current_expansion.module.mod_path.last()
                     .map(|id| id.as_str().to_string());
 

src/libsyntax/lib.rs

@@ -31,6 +31,8 @@ pub use rustc_data_structures::thin_vec::ThinVec;
 use ast::AttrId;
 use syntax_pos::edition::Edition;
 
+const MACRO_ARGUMENTS: Option<&'static str> = Some("macro arguments");
+
 // A variant of 'try!' that panics on an Err. This is used as a crutch on the
 // way towards a non-panic!-prone parser. It should be used for fatal parsing
 // errors; eventually we plan to convert all code using panictry to just use

src/libsyntax/parse/diagnostics.rs

@@ -13,7 +13,7 @@ use crate::symbol::kw;
 use crate::ThinVec;
 use errors::{Applicability, DiagnosticBuilder};
 use log::debug;
-use syntax_pos::Span;
+use syntax_pos::{Span, DUMMY_SP};
 
 pub trait RecoverQPath: Sized + 'static {
     const PATH_STYLE: PathStyle = PathStyle::Expr;
@@ -201,7 +201,7 @@ impl<'a> Parser<'a> {
 
         let mut path = ast::Path {
             segments: Vec::new(),
-            span: syntax_pos::DUMMY_SP,
+            span: DUMMY_SP,
         };
         self.parse_path_segments(&mut path.segments, T::PATH_STYLE)?;
         path.span = ty_span.to(self.prev_span);
@@ -267,6 +267,58 @@ impl<'a> Parser<'a> {
         }
     }
 
+    /// Create a `DiagnosticBuilder` for an unexpected token `t` and try to recover if it is a
+    /// closing delimiter.
+    pub fn unexpected_try_recover(
+        &mut self,
+        t: &token::Token,
+    ) -> PResult<'a, bool /* recovered */> {
+        let token_str = pprust::token_to_string(t);
+        let this_token_str = self.this_token_descr();
+        let (prev_sp, sp) = match (&self.token, self.subparser_name) {
+            // Point at the end of the macro call when reaching end of macro arguments.
+            (token::Token::Eof, Some(_)) => {
+                let sp = self.sess.source_map().next_point(self.span);
+                (sp, sp)
+            }
+            // We don't want to point at the following span after DUMMY_SP.
+            // This happens when the parser finds an empty TokenStream.
+            _ if self.prev_span == DUMMY_SP => (self.span, self.span),
+            // EOF, don't want to point at the following char, but rather the last token.
+            (token::Token::Eof, None) => (self.prev_span, self.span),
+            _ => (self.sess.source_map().next_point(self.prev_span), self.span),
+        };
+        let msg = format!(
+            "expected `{}`, found {}",
+            token_str,
+            match (&self.token, self.subparser_name) {
+                (token::Token::Eof, Some(origin)) => format!("end of {}", origin),
+                _ => this_token_str,
+            },
+        );
+        let mut err = self.struct_span_err(sp, &msg);
+        let label_exp = format!("expected `{}`", token_str);
+        match self.recover_closing_delimiter(&[t.clone()], err) {
+            Err(e) => err = e,
+            Ok(recovered) => {
+                return Ok(recovered);
+            }
+        }
+        let cm = self.sess.source_map();
+        match (cm.lookup_line(prev_sp.lo()), cm.lookup_line(sp.lo())) {
+            (Ok(ref a), Ok(ref b)) if a.line == b.line => {
+                // When the spans are in the same line, it means that the only content
+                // between them is whitespace, point only at the found token.
+                err.span_label(sp, label_exp);
+            }
+            _ => {
+                err.span_label(prev_sp, label_exp);
+                err.span_label(sp, "unexpected token");
+            }
+        }
+        Err(err)
+    }
+
     /// Consume alternative await syntaxes like `await <expr>`, `await? <expr>`, `await(<expr>)`
     /// and `await { <expr> }`.
     crate fn parse_incorrect_await_syntax(
@@ -562,4 +614,23 @@ impl<'a> Parser<'a> {
         }
     }
 
+    crate fn expected_expression_found(&self) -> DiagnosticBuilder<'a> {
+        let (span, msg) = match (&self.token, self.subparser_name) {
+            (&token::Token::Eof, Some(origin)) => {
+                let sp = self.sess.source_map().next_point(self.span);
+                (sp, format!("expected expression, found end of {}", origin))
+            }
+            _ => (self.span, format!(
+                "expected expression, found {}",
+                self.this_token_descr(),
+            )),
+        };
+        let mut err = self.struct_span_err(span, &msg);
+        let sp = self.sess.source_map().start_point(self.span);
+        if let Some(sp) = self.sess.ambiguous_block_expr_parse.borrow().get(&sp) {
+            self.sess.expr_parentheses_needed(&mut err, *sp, None);
+        }
+        err.span_label(span, "expected expression");
+        err
+    }
 }

src/libsyntax/parse/mod.rs

@@ -236,7 +236,7 @@ fn maybe_source_file_to_parser(
 ) -> Result<Parser<'_>, Vec<Diagnostic>> {
     let end_pos = source_file.end_pos;
     let (stream, unclosed_delims) = maybe_file_to_stream(sess, source_file, None)?;
-    let mut parser = stream_to_parser(sess, stream);
+    let mut parser = stream_to_parser(sess, stream, None);
     parser.unclosed_delims = unclosed_delims;
     if parser.token == token::Eof && parser.span.is_dummy() {
         parser.span = Span::new(end_pos, end_pos, parser.span.ctxt());
@@ -248,7 +248,7 @@ fn maybe_source_file_to_parser(
 // must preserve old name for now, because quote! from the *existing*
 // compiler expands into it
 pub fn new_parser_from_tts(sess: &ParseSess, tts: Vec<TokenTree>) -> Parser<'_> {
-    stream_to_parser(sess, tts.into_iter().collect())
+    stream_to_parser(sess, tts.into_iter().collect(), crate::MACRO_ARGUMENTS)
 }
 
 
@@ -328,8 +328,12 @@ pub fn maybe_file_to_stream(
 }
 
 /// Given stream and the `ParseSess`, produces a parser.
-pub fn stream_to_parser(sess: &ParseSess, stream: TokenStream) -> Parser<'_> {
-    Parser::new(sess, stream, None, true, false)
+pub fn stream_to_parser<'a>(
+    sess: &'a ParseSess,
+    stream: TokenStream,
+    subparser_name: Option<&'static str>,
+) -> Parser<'a> {
+    Parser::new(sess, stream, None, true, false, subparser_name)
 }
 
 /// Given stream, the `ParseSess` and the base directory, produces a parser.
@@ -343,10 +347,12 @@ pub fn stream_to_parser(sess: &ParseSess, stream: TokenStream) -> Parser<'_> {
 /// The main usage of this function is outside of rustc, for those who uses
 /// libsyntax as a library. Please do not remove this function while refactoring
 /// just because it is not used in rustc codebase!
-pub fn stream_to_parser_with_base_dir<'a>(sess: &'a ParseSess,
-                                          stream: TokenStream,
-                                          base_dir: Directory<'a>) -> Parser<'a> {
-    Parser::new(sess, stream, Some(base_dir), true, false)
+pub fn stream_to_parser_with_base_dir<'a>(
+    sess: &'a ParseSess,
+    stream: TokenStream,
+    base_dir: Directory<'a>,
+) -> Parser<'a> {
+    Parser::new(sess, stream, Some(base_dir), true, false, None)
 }
 
 /// A sequence separator.