diff --git a/Cargo.lock b/Cargo.lock index 6ce5458ed7a5d..19ecd2023c639 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4178,6 +4178,7 @@ dependencies = [ "rustc_lexer", "rustc_session", "rustc_span", + "smallvec 1.4.0", "unicode-normalization", ] diff --git a/src/librustc_ast/token.rs b/src/librustc_ast/token.rs index a5b9c2a95bbea..2e2bc380e844f 100644 --- a/src/librustc_ast/token.rs +++ b/src/librustc_ast/token.rs @@ -673,62 +673,6 @@ impl Token { Some(Token::new(kind, self.span.to(joint.span))) } - - // See comments in `Nonterminal::to_tokenstream` for why we care about - // *probably* equal here rather than actual equality - crate fn probably_equal_for_proc_macro(&self, other: &Token) -> bool { - if mem::discriminant(&self.kind) != mem::discriminant(&other.kind) { - return false; - } - match (&self.kind, &other.kind) { - (&Eq, &Eq) - | (&Lt, &Lt) - | (&Le, &Le) - | (&EqEq, &EqEq) - | (&Ne, &Ne) - | (&Ge, &Ge) - | (&Gt, &Gt) - | (&AndAnd, &AndAnd) - | (&OrOr, &OrOr) - | (&Not, &Not) - | (&Tilde, &Tilde) - | (&At, &At) - | (&Dot, &Dot) - | (&DotDot, &DotDot) - | (&DotDotDot, &DotDotDot) - | (&DotDotEq, &DotDotEq) - | (&Comma, &Comma) - | (&Semi, &Semi) - | (&Colon, &Colon) - | (&ModSep, &ModSep) - | (&RArrow, &RArrow) - | (&LArrow, &LArrow) - | (&FatArrow, &FatArrow) - | (&Pound, &Pound) - | (&Dollar, &Dollar) - | (&Question, &Question) - | (&Whitespace, &Whitespace) - | (&Comment, &Comment) - | (&Eof, &Eof) => true, - - (&BinOp(a), &BinOp(b)) | (&BinOpEq(a), &BinOpEq(b)) => a == b, - - (&OpenDelim(a), &OpenDelim(b)) | (&CloseDelim(a), &CloseDelim(b)) => a == b, - - (&DocComment(a), &DocComment(b)) | (&Shebang(a), &Shebang(b)) => a == b, - - (&Literal(a), &Literal(b)) => a == b, - - (&Lifetime(a), &Lifetime(b)) => a == b, - (&Ident(a, b), &Ident(c, d)) => { - b == d && (a == c || a == kw::DollarCrate || c == kw::DollarCrate) - } - - (&Interpolated(_), &Interpolated(_)) => false, - - _ => panic!("forgot to add a token?"), - } - } } impl PartialEq for Token { diff --git a/src/librustc_ast/tokenstream.rs b/src/librustc_ast/tokenstream.rs index 075aaa7e5bc01..9d0199078fa6a 100644 --- a/src/librustc_ast/tokenstream.rs +++ b/src/librustc_ast/tokenstream.rs @@ -21,8 +21,6 @@ use rustc_macros::HashStable_Generic; use rustc_span::{Span, DUMMY_SP}; use smallvec::{smallvec, SmallVec}; -use log::debug; - use std::{iter, mem}; /// When the main rust parser encounters a syntax-extension invocation, it @@ -68,23 +66,6 @@ impl TokenTree { } } - // See comments in `Nonterminal::to_tokenstream` for why we care about - // *probably* equal here rather than actual equality - // - // This is otherwise the same as `eq_unspanned`, only recursing with a - // different method. - pub fn probably_equal_for_proc_macro(&self, other: &TokenTree) -> bool { - match (self, other) { - (TokenTree::Token(token), TokenTree::Token(token2)) => { - token.probably_equal_for_proc_macro(token2) - } - (TokenTree::Delimited(_, delim, tts), TokenTree::Delimited(_, delim2, tts2)) => { - delim == delim2 && tts.probably_equal_for_proc_macro(&tts2) - } - _ => false, - } - } - /// Retrieves the TokenTree's span. pub fn span(&self) -> Span { match self { @@ -307,112 +288,6 @@ impl TokenStream { t1.next().is_none() && t2.next().is_none() } - // See comments in `Nonterminal::to_tokenstream` for why we care about - // *probably* equal here rather than actual equality - // - // This is otherwise the same as `eq_unspanned`, only recursing with a - // different method. - pub fn probably_equal_for_proc_macro(&self, other: &TokenStream) -> bool { - // When checking for `probably_eq`, we ignore certain tokens that aren't - // preserved in the AST. Because they are not preserved, the pretty - // printer arbitrarily adds or removes them when printing as token - // streams, making a comparison between a token stream generated from an - // AST and a token stream which was parsed into an AST more reliable. - fn semantic_tree(tree: &TokenTree) -> bool { - if let TokenTree::Token(token) = tree { - if let - // The pretty printer tends to add trailing commas to - // everything, and in particular, after struct fields. - | token::Comma - // The pretty printer emits `NoDelim` as whitespace. - | token::OpenDelim(DelimToken::NoDelim) - | token::CloseDelim(DelimToken::NoDelim) - // The pretty printer collapses many semicolons into one. - | token::Semi - // The pretty printer collapses whitespace arbitrarily and can - // introduce whitespace from `NoDelim`. - | token::Whitespace - // The pretty printer can turn `$crate` into `::crate_name` - | token::ModSep = token.kind { - return false; - } - } - true - } - - // When comparing two `TokenStream`s, we ignore the `IsJoint` information. - // - // However, `rustc_parse::lexer::tokentrees::TokenStreamBuilder` will - // use `Token.glue` on adjacent tokens with the proper `IsJoint`. - // Since we are ignoreing `IsJoint`, a 'glued' token (e.g. `BinOp(Shr)`) - // and its 'split'/'unglued' compoenents (e.g. `Gt, Gt`) are equivalent - // when determining if two `TokenStream`s are 'probably equal'. - // - // Therefore, we use `break_two_token_op` to convert all tokens - // to the 'unglued' form (if it exists). This ensures that two - // `TokenStream`s which differ only in how their tokens are glued - // will be considered 'probably equal', which allows us to keep spans. - // - // This is important when the original `TokenStream` contained - // extra spaces (e.g. `f :: < Vec < _ > > ( ) ;'). These extra spaces - // will be omitted when we pretty-print, which can cause the original - // and reparsed `TokenStream`s to differ in the assignment of `IsJoint`, - // leading to some tokens being 'glued' together in one stream but not - // the other. See #68489 for more details. - fn break_tokens(tree: TokenTree) -> impl Iterator { - // In almost all cases, we should have either zero or one levels - // of 'unglueing'. However, in some unusual cases, we may need - // to iterate breaking tokens mutliple times. For example: - // '[BinOpEq(Shr)] => [Gt, Ge] -> [Gt, Gt, Eq]' - let mut token_trees: SmallVec<[_; 2]>; - if let TokenTree::Token(token) = &tree { - let mut out = SmallVec::<[_; 2]>::new(); - out.push(token.clone()); - // Iterate to fixpoint: - // * We start off with 'out' containing our initial token, and `temp` empty - // * If we are able to break any tokens in `out`, then `out` will have - // at least one more element than 'temp', so we will try to break tokens - // again. - // * If we cannot break any tokens in 'out', we are done - loop { - let mut temp = SmallVec::<[_; 2]>::new(); - let mut changed = false; - - for token in out.into_iter() { - if let Some((first, second)) = token.kind.break_two_token_op() { - temp.push(Token::new(first, DUMMY_SP)); - temp.push(Token::new(second, DUMMY_SP)); - changed = true; - } else { - temp.push(token); - } - } - out = temp; - if !changed { - break; - } - } - token_trees = out.into_iter().map(|t| TokenTree::Token(t)).collect(); - if token_trees.len() != 1 { - debug!("break_tokens: broke {:?} to {:?}", tree, token_trees); - } - } else { - token_trees = SmallVec::new(); - token_trees.push(tree); - } - token_trees.into_iter() - } - - let mut t1 = self.trees().filter(semantic_tree).flat_map(break_tokens); - let mut t2 = other.trees().filter(semantic_tree).flat_map(break_tokens); - for (t1, t2) in t1.by_ref().zip(t2.by_ref()) { - if !t1.probably_equal_for_proc_macro(&t2) { - return false; - } - } - t1.next().is_none() && t2.next().is_none() - } - pub fn map_enumerated TokenTree>(self, mut f: F) -> TokenStream { TokenStream(Lrc::new( self.0 diff --git a/src/librustc_parse/Cargo.toml b/src/librustc_parse/Cargo.toml index 7164c67880863..0d31a8c7bc1fb 100644 --- a/src/librustc_parse/Cargo.toml +++ b/src/librustc_parse/Cargo.toml @@ -12,6 +12,7 @@ doctest = false [dependencies] bitflags = "1.0" log = "0.4" +smallvec = { version = "1.0", features = ["union", "may_dangle"] } rustc_ast_pretty = { path = "../librustc_ast_pretty" } rustc_data_structures = { path = "../librustc_data_structures" } rustc_feature = { path = "../librustc_feature" } diff --git a/src/librustc_parse/lib.rs b/src/librustc_parse/lib.rs index 8e2a9513d6b82..0c817a712819f 100644 --- a/src/librustc_parse/lib.rs +++ b/src/librustc_parse/lib.rs @@ -7,18 +7,22 @@ #![feature(or_patterns)] use rustc_ast::ast; -use rustc_ast::token::{self, Nonterminal}; -use rustc_ast::tokenstream::{self, TokenStream, TokenTree}; +use rustc_ast::token::{self, DelimToken, Nonterminal, Token, TokenKind}; +use rustc_ast::tokenstream::{self, IsJoint, TokenStream, TokenTree}; use rustc_ast_pretty::pprust; use rustc_data_structures::sync::Lrc; use rustc_errors::{Diagnostic, FatalError, Level, PResult}; use rustc_session::parse::ParseSess; -use rustc_span::{FileName, SourceFile, Span}; +use rustc_span::symbol::kw; +use rustc_span::{FileName, SourceFile, Span, DUMMY_SP}; +use smallvec::SmallVec; + +use std::mem; use std::path::Path; use std::str; -use log::info; +use log::{debug, info}; pub const MACRO_ARGUMENTS: Option<&'static str> = Some("macro arguments"); @@ -300,7 +304,7 @@ pub fn nt_to_tokenstream(nt: &Nonterminal, sess: &ParseSess, span: Span) -> Toke // modifications, including adding/removing typically non-semantic // tokens such as extra braces and commas, don't happen. if let Some(tokens) = tokens { - if tokens.probably_equal_for_proc_macro(&tokens_for_real) { + if tokenstream_probably_equal_for_proc_macro(&tokens, &tokens_for_real, sess) { return tokens; } info!( @@ -373,3 +377,203 @@ fn prepend_attrs( builder.push(tokens.clone()); Some(builder.build()) } + +// See comments in `Nonterminal::to_tokenstream` for why we care about +// *probably* equal here rather than actual equality +// +// This is otherwise the same as `eq_unspanned`, only recursing with a +// different method. +pub fn tokenstream_probably_equal_for_proc_macro( + first: &TokenStream, + other: &TokenStream, + sess: &ParseSess, +) -> bool { + // When checking for `probably_eq`, we ignore certain tokens that aren't + // preserved in the AST. Because they are not preserved, the pretty + // printer arbitrarily adds or removes them when printing as token + // streams, making a comparison between a token stream generated from an + // AST and a token stream which was parsed into an AST more reliable. + fn semantic_tree(tree: &TokenTree) -> bool { + if let TokenTree::Token(token) = tree { + if let + // The pretty printer tends to add trailing commas to + // everything, and in particular, after struct fields. + | token::Comma + // The pretty printer emits `NoDelim` as whitespace. + | token::OpenDelim(DelimToken::NoDelim) + | token::CloseDelim(DelimToken::NoDelim) + // The pretty printer collapses many semicolons into one. + | token::Semi + // The pretty printer collapses whitespace arbitrarily and can + // introduce whitespace from `NoDelim`. + | token::Whitespace + // The pretty printer can turn `$crate` into `::crate_name` + | token::ModSep = token.kind { + return false; + } + } + true + } + + // When comparing two `TokenStream`s, we ignore the `IsJoint` information. + // + // However, `rustc_parse::lexer::tokentrees::TokenStreamBuilder` will + // use `Token.glue` on adjacent tokens with the proper `IsJoint`. + // Since we are ignoreing `IsJoint`, a 'glued' token (e.g. `BinOp(Shr)`) + // and its 'split'/'unglued' compoenents (e.g. `Gt, Gt`) are equivalent + // when determining if two `TokenStream`s are 'probably equal'. + // + // Therefore, we use `break_two_token_op` to convert all tokens + // to the 'unglued' form (if it exists). This ensures that two + // `TokenStream`s which differ only in how their tokens are glued + // will be considered 'probably equal', which allows us to keep spans. + // + // This is important when the original `TokenStream` contained + // extra spaces (e.g. `f :: < Vec < _ > > ( ) ;'). These extra spaces + // will be omitted when we pretty-print, which can cause the original + // and reparsed `TokenStream`s to differ in the assignment of `IsJoint`, + // leading to some tokens being 'glued' together in one stream but not + // the other. See #68489 for more details. + fn break_tokens(tree: TokenTree) -> impl Iterator { + // In almost all cases, we should have either zero or one levels + // of 'unglueing'. However, in some unusual cases, we may need + // to iterate breaking tokens mutliple times. For example: + // '[BinOpEq(Shr)] => [Gt, Ge] -> [Gt, Gt, Eq]' + let mut token_trees: SmallVec<[_; 2]>; + if let TokenTree::Token(token) = &tree { + let mut out = SmallVec::<[_; 2]>::new(); + out.push(token.clone()); + // Iterate to fixpoint: + // * We start off with 'out' containing our initial token, and `temp` empty + // * If we are able to break any tokens in `out`, then `out` will have + // at least one more element than 'temp', so we will try to break tokens + // again. + // * If we cannot break any tokens in 'out', we are done + loop { + let mut temp = SmallVec::<[_; 2]>::new(); + let mut changed = false; + + for token in out.into_iter() { + if let Some((first, second)) = token.kind.break_two_token_op() { + temp.push(Token::new(first, DUMMY_SP)); + temp.push(Token::new(second, DUMMY_SP)); + changed = true; + } else { + temp.push(token); + } + } + out = temp; + if !changed { + break; + } + } + token_trees = out.into_iter().map(|t| TokenTree::Token(t)).collect(); + if token_trees.len() != 1 { + debug!("break_tokens: broke {:?} to {:?}", tree, token_trees); + } + } else { + token_trees = SmallVec::new(); + token_trees.push(tree); + } + token_trees.into_iter() + } + + let expand_nt = |tree: TokenTree| { + if let TokenTree::Token(Token { kind: TokenKind::Interpolated(nt), span }) = &tree { + nt_to_tokenstream(nt, sess, *span).into_trees() + } else { + TokenStream::new(vec![(tree, IsJoint::NonJoint)]).into_trees() + } + }; + + // Break tokens after we expand any nonterminals, so that we break tokens + // that are produced as a result of nonterminal expansion. + let mut t1 = first.trees().filter(semantic_tree).flat_map(expand_nt).flat_map(break_tokens); + let mut t2 = other.trees().filter(semantic_tree).flat_map(expand_nt).flat_map(break_tokens); + for (t1, t2) in t1.by_ref().zip(t2.by_ref()) { + if !tokentree_probably_equal_for_proc_macro(&t1, &t2, sess) { + return false; + } + } + t1.next().is_none() && t2.next().is_none() +} + +// See comments in `Nonterminal::to_tokenstream` for why we care about +// *probably* equal here rather than actual equality +crate fn token_probably_equal_for_proc_macro(first: &Token, other: &Token) -> bool { + use TokenKind::*; + + if mem::discriminant(&first.kind) != mem::discriminant(&other.kind) { + return false; + } + match (&first.kind, &other.kind) { + (&Eq, &Eq) + | (&Lt, &Lt) + | (&Le, &Le) + | (&EqEq, &EqEq) + | (&Ne, &Ne) + | (&Ge, &Ge) + | (&Gt, &Gt) + | (&AndAnd, &AndAnd) + | (&OrOr, &OrOr) + | (&Not, &Not) + | (&Tilde, &Tilde) + | (&At, &At) + | (&Dot, &Dot) + | (&DotDot, &DotDot) + | (&DotDotDot, &DotDotDot) + | (&DotDotEq, &DotDotEq) + | (&Comma, &Comma) + | (&Semi, &Semi) + | (&Colon, &Colon) + | (&ModSep, &ModSep) + | (&RArrow, &RArrow) + | (&LArrow, &LArrow) + | (&FatArrow, &FatArrow) + | (&Pound, &Pound) + | (&Dollar, &Dollar) + | (&Question, &Question) + | (&Whitespace, &Whitespace) + | (&Comment, &Comment) + | (&Eof, &Eof) => true, + + (&BinOp(a), &BinOp(b)) | (&BinOpEq(a), &BinOpEq(b)) => a == b, + + (&OpenDelim(a), &OpenDelim(b)) | (&CloseDelim(a), &CloseDelim(b)) => a == b, + + (&DocComment(a), &DocComment(b)) | (&Shebang(a), &Shebang(b)) => a == b, + + (&Literal(a), &Literal(b)) => a == b, + + (&Lifetime(a), &Lifetime(b)) => a == b, + (&Ident(a, b), &Ident(c, d)) => { + b == d && (a == c || a == kw::DollarCrate || c == kw::DollarCrate) + } + + // Expanded by `tokenstream_probably_equal_for_proc_macro` + (&Interpolated(_), &Interpolated(_)) => unreachable!(), + + _ => panic!("forgot to add a token?"), + } +} + +// See comments in `Nonterminal::to_tokenstream` for why we care about +// *probably* equal here rather than actual equality +// +// This is otherwise the same as `eq_unspanned`, only recursing with a +// different method. +pub fn tokentree_probably_equal_for_proc_macro( + first: &TokenTree, + other: &TokenTree, + sess: &ParseSess, +) -> bool { + match (first, other) { + (TokenTree::Token(token), TokenTree::Token(token2)) => { + token_probably_equal_for_proc_macro(token, token2) + } + (TokenTree::Delimited(_, delim, tts), TokenTree::Delimited(_, delim2, tts2)) => { + delim == delim2 && tokenstream_probably_equal_for_proc_macro(&tts, &tts2, sess) + } + _ => false, + } +} diff --git a/src/librustc_trait_selection/traits/select.rs b/src/librustc_trait_selection/traits/select.rs index b402aba65cdb7..9b3381066a17b 100644 --- a/src/librustc_trait_selection/traits/select.rs +++ b/src/librustc_trait_selection/traits/select.rs @@ -1058,20 +1058,11 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> { // Heuristics: show the diagnostics when there are no candidates in crate. if let Ok(candidate_set) = self.assemble_candidates(stack) { let mut no_candidates_apply = true; - { - let evaluated_candidates = - candidate_set.vec.iter().map(|c| self.evaluate_candidate(stack, &c)); - - for ec in evaluated_candidates { - match ec { - Ok(c) => { - if c.may_apply() { - no_candidates_apply = false; - break; - } - } - Err(e) => return Err(e.into()), - } + + for c in candidate_set.vec.iter() { + if self.evaluate_candidate(stack, &c)?.may_apply() { + no_candidates_apply = false; + break; } } @@ -3182,11 +3173,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> { assert_eq!(tys_a.len(), tys_b.len()); // The last field of the tuple has to exist. - let (&a_last, a_mid) = if let Some(x) = tys_a.split_last() { - x - } else { - return Err(Unimplemented); - }; + let (&a_last, a_mid) = tys_a.split_last().ok_or(Unimplemented)?; let &b_last = tys_b.last().unwrap(); // Check that the source tuple with the target's diff --git a/src/test/ui/proc-macro/macro-rules-capture.rs b/src/test/ui/proc-macro/macro-rules-capture.rs new file mode 100644 index 0000000000000..37436567d70f0 --- /dev/null +++ b/src/test/ui/proc-macro/macro-rules-capture.rs @@ -0,0 +1,18 @@ +// aux-build: test-macros.rs + +extern crate test_macros; +use test_macros::recollect_attr; + +macro_rules! reemit { + ($name:ident => $($token:expr)*) => { + + #[recollect_attr] + pub fn $name() { + $($token)*; + } + } +} + +reemit! { foo => 45u32.into() } //~ ERROR type annotations + +fn main() {} diff --git a/src/test/ui/proc-macro/macro-rules-capture.stderr b/src/test/ui/proc-macro/macro-rules-capture.stderr new file mode 100644 index 0000000000000..6d512846ff785 --- /dev/null +++ b/src/test/ui/proc-macro/macro-rules-capture.stderr @@ -0,0 +1,12 @@ +error[E0282]: type annotations needed + --> $DIR/macro-rules-capture.rs:16:24 + | +LL | reemit! { foo => 45u32.into() } + | ------^^^^-- + | | | + | | cannot infer type for type parameter `T` declared on the trait `Into` + | this method call resolves to `T` + +error: aborting due to previous error + +For more information about this error, try `rustc --explain E0282`.