Skip to content

Commit

Permalink
Rollup merge of rust-lang#37208 - jseyfried:fix_partially_consumed_to…
Browse files Browse the repository at this point in the history
…kens_in_macros, r=nrc

macros: fix partially consumed tokens in macro matchers

Fixes rust-lang#37175.

This PR also avoids re-transcribing the tokens consumed by a matcher (and cloning the `TtReader` once per matcher), which improves expansion performance of the test case from rust-lang#34630 by ~8%.

r? @nrc
  • Loading branch information
eddyb authored Oct 19, 2016
2 parents 4568318 + 95a9e2a commit 094eaf0
Show file tree
Hide file tree
Showing 4 changed files with 52 additions and 15 deletions.
21 changes: 9 additions & 12 deletions src/libsyntax/ext/tt/macro_parser.rs
Original file line number Diff line number Diff line change
Expand Up @@ -476,24 +476,21 @@ pub fn parse(sess: &ParseSess,
}
rdr.next_token();
} else /* bb_eis.len() == 1 */ {
let mut rust_parser = Parser::new(sess, cfg.clone(), Box::new(rdr.clone()));

let mut ei = bb_eis.pop().unwrap();
match ei.top_elts.get_tt(ei.idx) {
TokenTree::Token(span, MatchNt(_, ident)) => {
rdr.next_tok = {
let mut rust_parser = Parser::new(sess, cfg.clone(), Box::new(&mut rdr));
let mut ei = bb_eis.pop().unwrap();
if let TokenTree::Token(span, MatchNt(_, ident)) = ei.top_elts.get_tt(ei.idx) {
let match_cur = ei.match_cur;
(&mut ei.matches[match_cur]).push(Rc::new(MatchedNonterminal(
parse_nt(&mut rust_parser, span, &ident.name.as_str()))));
ei.idx += 1;
ei.match_cur += 1;
} else {
unreachable!()
}
_ => panic!()
}
cur_eis.push(ei);

for _ in 0..rust_parser.tokens_consumed {
let _ = rdr.next_token();
}
cur_eis.push(ei);
Some(TokenAndSpan { tok: rust_parser.token, sp: rust_parser.span })
};
}
}

Expand Down
5 changes: 5 additions & 0 deletions src/libsyntax/ext/tt/transcribe.rs
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ pub struct TtReader<'a> {
/* cached: */
pub cur_tok: Token,
pub cur_span: Span,
pub next_tok: Option<TokenAndSpan>,
/// Transform doc comments. Only useful in macro invocations
pub desugar_doc_comments: bool,
pub fatal_errs: Vec<DiagnosticBuilder<'a>>,
Expand Down Expand Up @@ -100,6 +101,7 @@ pub fn new_tt_reader_with_doc_flag(sp_diag: &Handler,
/* dummy values, never read: */
cur_tok: token::Eof,
cur_span: DUMMY_SP,
next_tok: None,
fatal_errs: Vec::new(),
};
tt_next_token(&mut r); /* get cur_tok and cur_span set up */
Expand Down Expand Up @@ -178,6 +180,9 @@ fn lockstep_iter_size(t: &TokenTree, r: &TtReader) -> LockstepIterSize {
/// Return the next token from the TtReader.
/// EFFECT: advances the reader's token field
pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan {
if let Some(tok) = r.next_tok.take() {
return tok;
}
// FIXME(pcwalton): Bad copy?
let ret_val = TokenAndSpan {
tok: r.cur_tok.clone(),
Expand Down
27 changes: 24 additions & 3 deletions src/libsyntax/parse/lexer/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,7 @@ impl<'a> Reader for StringReader<'a> {

impl<'a> Reader for TtReader<'a> {
fn is_eof(&self) -> bool {
self.cur_tok == token::Eof
self.peek().tok == token::Eof
}
fn try_next_token(&mut self) -> Result<TokenAndSpan, ()> {
assert!(self.fatal_errs.is_empty());
Expand All @@ -165,10 +165,31 @@ impl<'a> Reader for TtReader<'a> {
self.fatal_errs.clear();
}
fn peek(&self) -> TokenAndSpan {
TokenAndSpan {
self.next_tok.clone().unwrap_or(TokenAndSpan {
tok: self.cur_tok.clone(),
sp: self.cur_span,
}
})
}
}

impl<'a, 'b> Reader for &'b mut TtReader<'a> {
fn is_eof(&self) -> bool {
(**self).is_eof()
}
fn try_next_token(&mut self) -> Result<TokenAndSpan, ()> {
(**self).try_next_token()
}
fn fatal(&self, m: &str) -> FatalError {
(**self).fatal(m)
}
fn err(&self, m: &str) {
(**self).err(m)
}
fn emit_fatal_errors(&mut self) {
(**self).emit_fatal_errors()
}
fn peek(&self) -> TokenAndSpan {
(**self).peek()
}
}

Expand Down
14 changes: 14 additions & 0 deletions src/test/run-pass/issue-37175.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

macro_rules! m { (<$t:ty>) => { stringify!($t) } }
fn main() {
println!("{}", m!(<Vec<i32>>));
}

0 comments on commit 094eaf0

Please sign in to comment.