diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index 79058984b1352..1272643edd259 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -469,6 +469,18 @@ impl<'a> Builder<'a> { stage = compiler.stage; } + let mut extra_args = env::var(&format!("RUSTFLAGS_STAGE_{}", stage)).unwrap_or_default(); + if stage != 0 { + let s = env::var("RUSTFLAGS_STAGE_NOT_0").unwrap_or_default(); + extra_args.push_str(" "); + extra_args.push_str(&s); + } + + if !extra_args.is_empty() { + cargo.env("RUSTFLAGS", + format!("{} {}", env::var("RUSTFLAGS").unwrap_or_default(), extra_args)); + } + // Customize the compiler we're running. Specify the compiler to cargo // as our shim and then pass it some various options used to configure // how the actual compiler itself is called. diff --git a/src/doc/unstable-book/src/language-features/generators.md b/src/doc/unstable-book/src/language-features/generators.md index 7a559a7bec866..e8e2132dca254 100644 --- a/src/doc/unstable-book/src/language-features/generators.md +++ b/src/doc/unstable-book/src/language-features/generators.md @@ -139,11 +139,11 @@ closure-like semantics. Namely: types and such. * Traits like `Send` and `Sync` are automatically implemented for a `Generator` - depending on the captured variables of the environment. Unlike closures though + depending on the captured variables of the environment. Unlike closures, generators also depend on variables live across suspension points. This means that although the ambient environment may be `Send` or `Sync`, the generator itself may not be due to internal variables live across `yield` points being - not-`Send` or not-`Sync`. Note, though, that generators, like closures, do + not-`Send` or not-`Sync`. Note that generators, like closures, do not implement traits like `Copy` or `Clone` automatically. * Whenever a generator is dropped it will drop all captured environment @@ -155,7 +155,7 @@ lifted at a future date, the design is ongoing! ### Generators as state machines -In the compiler generators are currently compiled as state machines. Each +In the compiler, generators are currently compiled as state machines. Each `yield` expression will correspond to a different state that stores all live variables over that suspension point. Resumption of a generator will dispatch on the current state and then execute internally until a `yield` is reached, at diff --git a/src/liballoc/btree/map.rs b/src/liballoc/btree/map.rs index b114dc640fbaf..b320bed54320a 100644 --- a/src/liballoc/btree/map.rs +++ b/src/liballoc/btree/map.rs @@ -1748,6 +1748,11 @@ impl<'a, K: Ord, Q: ?Sized, V> Index<&'a Q> for BTreeMap { type Output = V; + /// Returns a reference to the value corresponding to the supplied key. + /// + /// # Panics + /// + /// Panics if the key is not present in the `BTreeMap`. #[inline] fn index(&self, key: &Q) -> &V { self.get(key).expect("no entry found for key") diff --git a/src/librustc_errors/emitter.rs b/src/librustc_errors/emitter.rs index 8a4fd24a29b89..ffb5efd93ed54 100644 --- a/src/librustc_errors/emitter.rs +++ b/src/librustc_errors/emitter.rs @@ -1014,8 +1014,21 @@ impl EmitterWriter { // Then, the secondary file indicator buffer.prepend(buffer_msg_line_offset + 1, "::: ", Style::LineNumber); + let loc = if let Some(first_line) = annotated_file.lines.first() { + let col = if let Some(first_annotation) = first_line.annotations.first() { + format!(":{}", first_annotation.start_col + 1) + } else { + "".to_string() + }; + format!("{}:{}{}", + annotated_file.file.name, + cm.doctest_offset_line(first_line.line_index), + col) + } else { + annotated_file.file.name.to_string() + }; buffer.append(buffer_msg_line_offset + 1, - &annotated_file.file.name.to_string(), + &loc, Style::LineAndColumn); for _ in 0..max_line_num_len { buffer.prepend(buffer_msg_line_offset + 1, " ", Style::NoStyle); diff --git a/src/librustc_errors/snippet.rs b/src/librustc_errors/snippet.rs index c2f4701999ea9..6035f33c822ce 100644 --- a/src/librustc_errors/snippet.rs +++ b/src/librustc_errors/snippet.rs @@ -27,7 +27,8 @@ pub struct FileInfo { /// The "primary file", if any, gets a `-->` marker instead of /// `>>>`, and has a line-number/column printed and not just a - /// filename. It appears first in the listing. It is known to + /// filename (other files are not guaranteed to have line numbers + /// or columns). It appears first in the listing. It is known to /// contain at least one primary span, though primary spans (which /// are designated with `^^^`) may also occur in other files. primary_span: Option, diff --git a/src/librustc_trans/llvm_util.rs b/src/librustc_trans/llvm_util.rs index 15988008de2fc..843231d376f6c 100644 --- a/src/librustc_trans/llvm_util.rs +++ b/src/librustc_trans/llvm_util.rs @@ -79,16 +79,16 @@ unsafe fn configure_llvm(sess: &Session) { // detection code will walk past the end of the feature array, // leading to crashes. -const ARM_WHITELIST: &'static [&'static str] = &["neon\0", "vfp2\0", "vfp3\0", "vfp4\0"]; +const ARM_WHITELIST: &'static [&'static str] = &["neon\0", "v7\0", "vfp2\0", "vfp3\0", "vfp4\0"]; -const AARCH64_WHITELIST: &'static [&'static str] = &["neon\0"]; +const AARCH64_WHITELIST: &'static [&'static str] = &["neon\0", "v7\0"]; const X86_WHITELIST: &'static [&'static str] = &["avx\0", "avx2\0", "bmi\0", "bmi2\0", "sse\0", "sse2\0", "sse3\0", "sse4.1\0", "sse4.2\0", "ssse3\0", "tbm\0", "lzcnt\0", "popcnt\0", "sse4a\0", "rdrnd\0", "rdseed\0", "fma\0", "xsave\0", "xsaveopt\0", "xsavec\0", - "xsaves\0", + "xsaves\0", "aes\0", "avx512bw\0", "avx512cd\0", "avx512dq\0", "avx512er\0", "avx512f\0", "avx512ifma\0", diff --git a/src/librustdoc/html/markdown.rs b/src/librustdoc/html/markdown.rs index dce0c4b001a0d..82ced00644da8 100644 --- a/src/librustdoc/html/markdown.rs +++ b/src/librustdoc/html/markdown.rs @@ -872,7 +872,7 @@ pub fn render(w: &mut fmt::Formatter, let link_out = format!("{content}", link = link_buf, title = title.map_or(String::new(), - |t| format!(" title=\"{}\"", t)), + |t| format!(" title=\"{}\"", Escape(&t))), content = content.unwrap_or(String::new())); unsafe { hoedown_buffer_put(ob, link_out.as_ptr(), link_out.len()); } diff --git a/src/libstd/collections/hash/map.rs b/src/libstd/collections/hash/map.rs index b01420f36a0c3..82a687ae5e493 100644 --- a/src/libstd/collections/hash/map.rs +++ b/src/libstd/collections/hash/map.rs @@ -1384,9 +1384,14 @@ impl<'a, K, Q: ?Sized, V, S> Index<&'a Q> for HashMap { type Output = V; + /// Returns a reference to the value corresponding to the supplied key. + /// + /// # Panics + /// + /// Panics if the key is not present in the `HashMap`. #[inline] - fn index(&self, index: &Q) -> &V { - self.get(index).expect("no entry found for key") + fn index(&self, key: &Q) -> &V { + self.get(key).expect("no entry found for key") } } diff --git a/src/libstd/process.rs b/src/libstd/process.rs index 5c66ac6ddded8..9b2f815b71383 100644 --- a/src/libstd/process.rs +++ b/src/libstd/process.rs @@ -1843,4 +1843,10 @@ mod tests { } assert!(events > 0); } + + #[test] + fn test_command_implements_send() { + fn take_send_type(_: T) {} + take_send_type(Command::new("")) + } } diff --git a/src/libstd/sys/unix/process/process_common.rs b/src/libstd/sys/unix/process/process_common.rs index c53bcdbf8e36f..7e057401fab70 100644 --- a/src/libstd/sys/unix/process/process_common.rs +++ b/src/libstd/sys/unix/process/process_common.rs @@ -45,7 +45,7 @@ pub struct Command { // other keys. program: CString, args: Vec, - argv: Vec<*const c_char>, + argv: Argv, env: CommandEnv, cwd: Option, @@ -58,6 +58,12 @@ pub struct Command { stderr: Option, } +// Create a new type for argv, so that we can make it `Send` +struct Argv(Vec<*const c_char>); + +// It is safe to make Argv Send, because it contains pointers to memory owned by `Command.args` +unsafe impl Send for Argv {} + // passed back to std::process with the pipes connected to the child, if any // were requested pub struct StdioPipes { @@ -92,7 +98,7 @@ impl Command { let mut saw_nul = false; let program = os2c(program, &mut saw_nul); Command { - argv: vec![program.as_ptr(), ptr::null()], + argv: Argv(vec![program.as_ptr(), ptr::null()]), program, args: Vec::new(), env: Default::default(), @@ -111,8 +117,8 @@ impl Command { // Overwrite the trailing NULL pointer in `argv` and then add a new null // pointer. let arg = os2c(arg, &mut self.saw_nul); - self.argv[self.args.len() + 1] = arg.as_ptr(); - self.argv.push(ptr::null()); + self.argv.0[self.args.len() + 1] = arg.as_ptr(); + self.argv.0.push(ptr::null()); // Also make sure we keep track of the owned value to schedule a // destructor for this memory. @@ -133,7 +139,7 @@ impl Command { self.saw_nul } pub fn get_argv(&self) -> &Vec<*const c_char> { - &self.argv + &self.argv.0 } #[allow(dead_code)] diff --git a/src/libsyntax/ext/tt/macro_parser.rs b/src/libsyntax/ext/tt/macro_parser.rs index 3e3c1618fffb2..1a9849ca5307d 100644 --- a/src/libsyntax/ext/tt/macro_parser.rs +++ b/src/libsyntax/ext/tt/macro_parser.rs @@ -90,8 +90,8 @@ use codemap::Spanned; use errors::FatalError; use ext::tt::quoted::{self, TokenTree}; use parse::{Directory, ParseSess}; -use parse::parser::{PathStyle, Parser}; -use parse::token::{self, DocComment, Token, Nonterminal}; +use parse::parser::{Parser, PathStyle}; +use parse::token::{self, DocComment, Nonterminal, Token}; use print::pprust; use symbol::keywords; use tokenstream::TokenStream; @@ -100,11 +100,12 @@ use util::small_vector::SmallVector; use std::mem; use std::rc::Rc; use std::collections::HashMap; -use std::collections::hash_map::Entry::{Vacant, Occupied}; +use std::collections::hash_map::Entry::{Occupied, Vacant}; -// To avoid costly uniqueness checks, we require that `MatchSeq` always has -// a nonempty body. +// To avoid costly uniqueness checks, we require that `MatchSeq` always has a nonempty body. +/// Either a sequence of token trees or a single one. This is used as the representation of the +/// sequence of tokens that make up a matcher. #[derive(Clone)] enum TokenTreeOrTokenTreeVec { Tt(TokenTree), @@ -112,6 +113,8 @@ enum TokenTreeOrTokenTreeVec { } impl TokenTreeOrTokenTreeVec { + /// Returns the number of constituent top-level token trees of `self` (top-level in that it + /// will not recursively descend into subtrees). fn len(&self) -> usize { match *self { TtSeq(ref v) => v.len(), @@ -119,6 +122,7 @@ impl TokenTreeOrTokenTreeVec { } } + /// The the `index`-th token tree of `self`. fn get_tt(&self, index: usize) -> TokenTree { match *self { TtSeq(ref v) => v[index].clone(), @@ -127,36 +131,96 @@ impl TokenTreeOrTokenTreeVec { } } -/// an unzipping of `TokenTree`s +/// An unzipping of `TokenTree`s... see the `stack` field of `MatcherPos`. +/// +/// This is used by `inner_parse_loop` to keep track of delimited submatchers that we have +/// descended into. #[derive(Clone)] struct MatcherTtFrame { + /// The "parent" matcher that we are descending into. elts: TokenTreeOrTokenTreeVec, + /// The position of the "dot" in `elts` at the time we descended. idx: usize, } +/// Represents a single "position" (aka "matcher position", aka "item"), as described in the module +/// documentation. #[derive(Clone)] struct MatcherPos { - stack: Vec, + /// The token or sequence of tokens that make up the matcher top_elts: TokenTreeOrTokenTreeVec, - sep: Option, + /// The position of the "dot" in this matcher idx: usize, - up: Option>, + /// The beginning position in the source that the beginning of this matcher corresponds to. In + /// other words, the token in the source at `sp_lo` is matched against the first token of the + /// matcher. + sp_lo: BytePos, + + /// For each named metavar in the matcher, we keep track of token trees matched against the + /// metavar by the black box parser. In particular, there may be more than one match per + /// metavar if we are in a repetition (each repetition matches each of the variables). + /// Moreover, matchers and repetitions can be nested; the `matches` field is shared (hence the + /// `Rc`) among all "nested" matchers. `match_lo`, `match_cur`, and `match_hi` keep track of + /// the current position of the `self` matcher position in the shared `matches` list. + /// + /// Also, note that while we are descending into a sequence, matchers are given their own + /// `matches` vector. Only once we reach the end of a full repetition of the sequence do we add + /// all bound matches from the submatcher into the shared top-level `matches` vector. If `sep` + /// and `up` are `Some`, then `matches` is _not_ the shared top-level list. Instead, if one + /// wants the shared `matches`, one should use `up.matches`. matches: Vec>>, + /// The position in `matches` corresponding to the first metavar in this matcher's sequence of + /// token trees. In other words, the first metavar in the first token of `top_elts` corresponds + /// to `matches[match_lo]`. match_lo: usize, + /// The position in `matches` corresponding to the metavar we are currently trying to match + /// against the source token stream. `match_lo <= match_cur <= match_hi`. match_cur: usize, + /// Similar to `match_lo` except `match_hi` is the position in `matches` of the _last_ metavar + /// in this matcher. match_hi: usize, - sp_lo: BytePos, + + // Specifically used if we are matching a repetition. If we aren't both should be `None`. + /// The separator if we are in a repetition + sep: Option, + /// The "parent" matcher position if we are in a repetition. That is, the matcher position just + /// before we enter the sequence. + up: Option>, + + // Specifically used to "unzip" token trees. By "unzip", we mean to unwrap the delimiters from + // a delimited token tree (e.g. something wrapped in `(` `)`) or to get the contents of a doc + // comment... + /// When matching against matchers with nested delimited submatchers (e.g. `pat ( pat ( .. ) + /// pat ) pat`), we need to keep track of the matchers we are descending into. This stack does + /// that where the bottom of the stack is the outermost matcher. + // Also, throughout the comments, this "descent" is often referred to as "unzipping"... + stack: Vec, } impl MatcherPos { + /// Add `m` as a named match for the `idx`-th metavar. fn push_match(&mut self, idx: usize, m: NamedMatch) { let matches = Rc::make_mut(&mut self.matches[idx]); matches.push(m); } } +/// Represents the possible results of an attempted parse. +pub enum ParseResult { + /// Parsed successfully. + Success(T), + /// Arm failed to match. If the second parameter is `token::Eof`, it indicates an unexpected + /// end of macro invocation. Otherwise, it indicates that no rules expected the given token. + Failure(syntax_pos::Span, Token), + /// Fatal error (malformed macro?). Abort compilation. + Error(syntax_pos::Span, String), +} + +/// A `ParseResult` where the `Success` variant contains a mapping of `Ident`s to `NamedMatch`es. +/// This represents the mapping of metavars to the token trees they bind to. pub type NamedParseResult = ParseResult>>; +/// Count how many metavars are named in the given matcher `ms`. pub fn count_names(ms: &[TokenTree]) -> usize { ms.iter().fold(0, |count, elt| { count + match *elt { @@ -169,20 +233,38 @@ pub fn count_names(ms: &[TokenTree]) -> usize { }) } +/// Initialize `len` empty shared `Vec`s to be used to store matches of metavars. +fn create_matches(len: usize) -> Vec>> { + (0..len).into_iter().map(|_| Rc::new(Vec::new())).collect() +} + +/// Generate the top-level matcher position in which the "dot" is before the first token of the +/// matcher `ms` and we are going to start matching at position `lo` in the source. fn initial_matcher_pos(ms: Vec, lo: BytePos) -> Box { let match_idx_hi = count_names(&ms[..]); let matches = create_matches(match_idx_hi); Box::new(MatcherPos { - stack: vec![], - top_elts: TtSeq(ms), - sep: None, + // Start with the top level matcher given to us + top_elts: TtSeq(ms), // "elts" is an abbr. for "elements" + // The "dot" is before the first token of the matcher idx: 0, - up: None, + // We start matching with byte `lo` in the source code + sp_lo: lo, + + // Initialize `matches` to a bunch of empty `Vec`s -- one for each metavar in `top_elts`. + // `match_lo` for `top_elts` is 0 and `match_hi` is `matches.len()`. `match_cur` is 0 since + // we haven't actually matched anything yet. matches, match_lo: 0, match_cur: 0, match_hi: match_idx_hi, - sp_lo: lo + + // Haven't descended into any delimiters, so empty stack + stack: vec![], + + // Haven't descended into any sequences, so both of these are `None`. + sep: None, + up: None, }) } @@ -202,29 +284,36 @@ fn initial_matcher_pos(ms: Vec, lo: BytePos) -> Box { /// token tree. The depth of the `NamedMatch` structure will therefore depend /// only on the nesting depth of `ast::TTSeq`s in the originating /// token tree it was derived from. - #[derive(Debug, Clone)] pub enum NamedMatch { MatchedSeq(Rc>, syntax_pos::Span), - MatchedNonterminal(Rc) + MatchedNonterminal(Rc), } -fn nameize>(sess: &ParseSess, ms: &[TokenTree], mut res: I) - -> NamedParseResult { - fn n_rec>(sess: &ParseSess, m: &TokenTree, res: &mut I, - ret_val: &mut HashMap>) - -> Result<(), (syntax_pos::Span, String)> { +/// Takes a sequence of token trees `ms` representing a matcher which successfully matched input +/// and an iterator of items that matched input and produces a `NamedParseResult`. +fn nameize>( + sess: &ParseSess, + ms: &[TokenTree], + mut res: I, +) -> NamedParseResult { + // Recursively descend into each type of matcher (e.g. sequences, delimited, metavars) and make + // sure that each metavar has _exactly one_ binding. If a metavar does not have exactly one + // binding, then there is an error. If it does, then we insert the binding into the + // `NamedParseResult`. + fn n_rec>( + sess: &ParseSess, + m: &TokenTree, + res: &mut I, + ret_val: &mut HashMap>, + ) -> Result<(), (syntax_pos::Span, String)> { match *m { - TokenTree::Sequence(_, ref seq) => { - for next_m in &seq.tts { - n_rec(sess, next_m, res.by_ref(), ret_val)? - } - } - TokenTree::Delimited(_, ref delim) => { - for next_m in &delim.tts { - n_rec(sess, next_m, res.by_ref(), ret_val)?; - } - } + TokenTree::Sequence(_, ref seq) => for next_m in &seq.tts { + n_rec(sess, next_m, res.by_ref(), ret_val)? + }, + TokenTree::Delimited(_, ref delim) => for next_m in &delim.tts { + n_rec(sess, next_m, res.by_ref(), ret_val)?; + }, TokenTree::MetaVarDecl(span, _, id) if id.name == keywords::Invalid.name() => { if sess.missing_fragment_specifiers.borrow_mut().remove(&span) { return Err((span, "missing fragment specifier".to_string())); @@ -250,7 +339,7 @@ fn nameize>(sess: &ParseSess, ms: &[TokenTree], mut let mut ret_val = HashMap::new(); for m in ms { match n_rec(sess, m, res.by_ref(), &mut ret_val) { - Ok(_) => {}, + Ok(_) => {} Err((sp, msg)) => return Error(sp, msg), } } @@ -258,25 +347,20 @@ fn nameize>(sess: &ParseSess, ms: &[TokenTree], mut Success(ret_val) } -pub enum ParseResult { - Success(T), - /// Arm failed to match. If the second parameter is `token::Eof`, it - /// indicates an unexpected end of macro invocation. Otherwise, it - /// indicates that no rules expected the given token. - Failure(syntax_pos::Span, Token), - /// Fatal error (malformed macro?). Abort compilation. - Error(syntax_pos::Span, String) -} - +/// Generate an appropriate parsing failure message. For EOF, this is "unexpected end...". For +/// other tokens, this is "unexpected token...". pub fn parse_failure_msg(tok: Token) -> String { match tok { token::Eof => "unexpected end of macro invocation".to_string(), - _ => format!("no rules expected the token `{}`", pprust::token_to_string(&tok)), + _ => format!( + "no rules expected the token `{}`", + pprust::token_to_string(&tok) + ), } } /// Perform a token equality check, ignoring syntax context (that is, an unhygienic comparison) -fn token_name_eq(t1 : &Token, t2 : &Token) -> bool { +fn token_name_eq(t1: &Token, t2: &Token) -> bool { if let (Some(id1), Some(id2)) = (t1.ident(), t2.ident()) { id1.name == id2.name } else if let (&token::Lifetime(id1), &token::Lifetime(id2)) = (t1, t2) { @@ -286,77 +370,121 @@ fn token_name_eq(t1 : &Token, t2 : &Token) -> bool { } } -fn create_matches(len: usize) -> Vec>> { - (0..len).into_iter().map(|_| Rc::new(Vec::new())).collect() -} - -fn inner_parse_loop(sess: &ParseSess, - cur_items: &mut SmallVector>, - next_items: &mut Vec>, - eof_items: &mut SmallVector>, - bb_items: &mut SmallVector>, - token: &Token, - span: syntax_pos::Span) - -> ParseResult<()> { +/// Process the matcher positions of `cur_items` until it is empty. In the process, this will +/// produce more items in `next_items`, `eof_items`, and `bb_items`. +/// +/// For more info about the how this happens, see the module-level doc comments and the inline +/// comments of this function. +/// +/// # Parameters +/// +/// - `sess`: the parsing session into which errors are emitted. +/// - `cur_items`: the set of current items to be processed. This should be empty by the end of a +/// successful execution of this function. +/// - `next_items`: the set of newly generated items. These are used to replenish `cur_items` in +/// the function `parse`. +/// - `eof_items`: the set of items that would be valid if this was the EOF. +/// - `bb_items`: the set of items that are waiting for the black-box parser. +/// - `token`: the current token of the parser. +/// - `span`: the `Span` in the source code corresponding to the token trees we are trying to match +/// against the matcher positions in `cur_items`. +/// +/// # Returns +/// +/// A `ParseResult`. Note that matches are kept track of through the items generated. +fn inner_parse_loop( + sess: &ParseSess, + cur_items: &mut SmallVector>, + next_items: &mut Vec>, + eof_items: &mut SmallVector>, + bb_items: &mut SmallVector>, + token: &Token, + span: syntax_pos::Span, +) -> ParseResult<()> { + // Pop items from `cur_items` until it is empty. while let Some(mut item) = cur_items.pop() { - // When unzipped trees end, remove them + // When unzipped trees end, remove them. This corresponds to backtracking out of a + // delimited submatcher into which we already descended. In backtracking out again, we need + // to advance the "dot" past the delimiters in the outer matcher. while item.idx >= item.top_elts.len() { match item.stack.pop() { Some(MatcherTtFrame { elts, idx }) => { item.top_elts = elts; item.idx = idx + 1; } - None => break + None => break, } } + // Get the current position of the "dot" (`idx`) in `item` and the number of token trees in + // the matcher (`len`). let idx = item.idx; let len = item.top_elts.len(); - // at end of sequence + // If `idx >= len`, then we are at or past the end of the matcher of `item`. if idx >= len { - // We are repeating iff there is a parent + // We are repeating iff there is a parent. If the matcher is inside of a repetition, + // then we could be at the end of a sequence or at the beginning of the next + // repetition. if item.up.is_some() { - // Disregarding the separator, add the "up" case to the tokens that should be - // examined. - // (remove this condition to make trailing seps ok) + // At this point, regardless of whether there is a separator, we should add all + // matches from the complete repetition of the sequence to the shared, top-level + // `matches` list (actually, `up.matches`, which could itself not be the top-level, + // but anyway...). Moreover, we add another item to `cur_items` in which the "dot" + // is at the end of the `up` matcher. This ensures that the "dot" in the `up` + // matcher is also advanced sufficiently. + // + // NOTE: removing the condition `idx == len` allows trailing separators. if idx == len { + // Get the `up` matcher let mut new_pos = item.up.clone().unwrap(); - // update matches (the MBE "parse tree") by appending - // each tree as a subtree. - - // Only touch the binders we have actually bound + // Add matches from this repetition to the `matches` of `up` for idx in item.match_lo..item.match_hi { let sub = item.matches[idx].clone(); let span = span.with_lo(item.sp_lo); new_pos.push_match(idx, MatchedSeq(sub, span)); } + // Move the "dot" past the repetition in `up` new_pos.match_cur = item.match_hi; new_pos.idx += 1; cur_items.push(new_pos); } - // Check if we need a separator + // Check if we need a separator. if idx == len && item.sep.is_some() { - // We have a separator, and it is the current token. - if item.sep.as_ref().map(|sep| token_name_eq(token, sep)).unwrap_or(false) { + // We have a separator, and it is the current token. We can advance past the + // separator token. + if item.sep + .as_ref() + .map(|sep| token_name_eq(token, sep)) + .unwrap_or(false) + { item.idx += 1; next_items.push(item); } - } else { // we don't need a separator + } + // We don't need a separator. Move the "dot" back to the beginning of the matcher + // and try to match again. + else { item.match_cur = item.match_lo; item.idx = 0; cur_items.push(item); } - } else { - // We aren't repeating, so we must be potentially at the end of the input. + } + // If we are not in a repetition, then being at the end of a matcher means that we have + // reached the potential end of the input. + else { eof_items.push(item); } - } else { + } + // We are in the middle of a matcher. + else { + // Look at what token in the matcher we are trying to match the current token (`token`) + // against. Depending on that, we may generate new items. match item.top_elts.get_tt(idx) { - /* need to descend into sequence */ + // Need to descend into a sequence TokenTree::Sequence(sp, seq) => { if seq.op == quoted::KleeneOp::ZeroOrMore { // Examine the case where there are 0 matches of this sequence @@ -384,11 +512,16 @@ fn inner_parse_loop(sess: &ParseSess, top_elts: Tt(TokenTree::Sequence(sp, seq)), })); } + + // We need to match a metavar (but the identifier is invalid)... this is an error TokenTree::MetaVarDecl(span, _, id) if id.name == keywords::Invalid.name() => { if sess.missing_fragment_specifiers.borrow_mut().remove(&span) { return Error(span, "missing fragment specifier".to_string()); } } + + // We need to match a metavar with a valid ident... call out to the black-box + // parser by adding an item to `bb_items`. TokenTree::MetaVarDecl(_, _, id) => { // Built-in nonterminals never start with these tokens, // so we can eliminate them from consideration. @@ -396,6 +529,13 @@ fn inner_parse_loop(sess: &ParseSess, bb_items.push(item); } } + + // We need to descend into a delimited submatcher or a doc comment. To do this, we + // push the current matcher onto a stack and push a new item containing the + // submatcher onto `cur_items`. + // + // At the beginning of the loop, if we reach the end of the delimited submatcher, + // we pop the stack to backtrack out of the descent. seq @ TokenTree::Delimited(..) | seq @ TokenTree::Token(_, DocComment(..)) => { let lower_elts = mem::replace(&mut item.top_elts, Tt(seq)); let idx = item.idx; @@ -406,36 +546,76 @@ fn inner_parse_loop(sess: &ParseSess, item.idx = 0; cur_items.push(item); } + + // We just matched a normal token. We can just advance the parser. TokenTree::Token(_, ref t) if token_name_eq(t, token) => { item.idx += 1; next_items.push(item); } + + // There was another token that was not `token`... This means we can't add any + // rules. NOTE that this is not necessarily an error unless _all_ items in + // `cur_items` end up doing this. There may still be some other matchers that do + // end up working out. TokenTree::Token(..) | TokenTree::MetaVar(..) => {} } } } + // Yay a successful parse (so far)! Success(()) } -pub fn parse(sess: &ParseSess, - tts: TokenStream, - ms: &[TokenTree], - directory: Option, - recurse_into_modules: bool) - -> NamedParseResult { +/// Use the given sequence of token trees (`ms`) as a matcher. Match the given token stream `tts` +/// against it and return the match. +/// +/// # Parameters +/// +/// - `sess`: The session into which errors are emitted +/// - `tts`: The tokenstream we are matching against the pattern `ms` +/// - `ms`: A sequence of token trees representing a pattern against which we are matching +/// - `directory`: Information about the file locations (needed for the black-box parser) +/// - `recurse_into_modules`: Whether or not to recurse into modules (needed for the black-box +/// parser) +pub fn parse( + sess: &ParseSess, + tts: TokenStream, + ms: &[TokenTree], + directory: Option, + recurse_into_modules: bool, +) -> NamedParseResult { + // Create a parser that can be used for the "black box" parts. let mut parser = Parser::new(sess, tts, directory, recurse_into_modules, true); + + // A queue of possible matcher positions. We initialize it with the matcher position in which + // the "dot" is before the first token of the first token tree in `ms`. `inner_parse_loop` then + // processes all of these possible matcher positions and produces posible next positions into + // `next_items`. After some post-processing, the contents of `next_items` replenish `cur_items` + // and we start over again. let mut cur_items = SmallVector::one(initial_matcher_pos(ms.to_owned(), parser.span.lo())); - let mut next_items = Vec::new(); // or proceed normally + let mut next_items = Vec::new(); loop { - let mut bb_items = SmallVector::new(); // black-box parsed by parser.rs + // Matcher positions black-box parsed by parser.rs (`parser`) + let mut bb_items = SmallVector::new(); + + // Matcher positions that would be valid if the macro invocation was over now let mut eof_items = SmallVector::new(); assert!(next_items.is_empty()); - match inner_parse_loop(sess, &mut cur_items, &mut next_items, &mut eof_items, &mut bb_items, - &parser.token, parser.span) { - Success(_) => {}, + // Process `cur_items` until either we have finished the input or we need to get some + // parsing from the black-box parser done. The result is that `next_items` will contain a + // bunch of possible next matcher positions in `next_items`. + match inner_parse_loop( + sess, + &mut cur_items, + &mut next_items, + &mut eof_items, + &mut bb_items, + &parser.token, + parser.span, + ) { + Success(_) => {} Failure(sp, tok) => return Failure(sp, tok), Error(sp, msg) => return Error(sp, msg), } @@ -443,46 +623,75 @@ pub fn parse(sess: &ParseSess, // inner parse loop handled all cur_items, so it's empty assert!(cur_items.is_empty()); - /* error messages here could be improved with links to orig. rules */ + // We need to do some post processing after the `inner_parser_loop`. + // + // Error messages here could be improved with links to original rules. + + // If we reached the EOF, check that there is EXACTLY ONE possible matcher. Otherwise, + // either the parse is ambiguous (which should never happen) or their is a syntax error. if token_name_eq(&parser.token, &token::Eof) { if eof_items.len() == 1 { - let matches = eof_items[0].matches.iter_mut().map(|dv| { - Rc::make_mut(dv).pop().unwrap() - }); + let matches = eof_items[0] + .matches + .iter_mut() + .map(|dv| Rc::make_mut(dv).pop().unwrap()); return nameize(sess, ms, matches); } else if eof_items.len() > 1 { - return Error(parser.span, "ambiguity: multiple successful parses".to_string()); + return Error( + parser.span, + "ambiguity: multiple successful parses".to_string(), + ); } else { return Failure(parser.span, token::Eof); } - } else if (!bb_items.is_empty() && !next_items.is_empty()) || bb_items.len() > 1 { - let nts = bb_items.iter().map(|item| match item.top_elts.get_tt(item.idx) { - TokenTree::MetaVarDecl(_, bind, name) => { - format!("{} ('{}')", name, bind) - } - _ => panic!() - }).collect::>().join(" or "); - - return Error(parser.span, format!( - "local ambiguity: multiple parsing options: {}", - match next_items.len() { - 0 => format!("built-in NTs {}.", nts), - 1 => format!("built-in NTs {} or 1 other option.", nts), - n => format!("built-in NTs {} or {} other options.", nts, n), - } - )); - } else if bb_items.is_empty() && next_items.is_empty() { + } + // Another possibility is that we need to call out to parse some rust nonterminal + // (black-box) parser. However, if there is not EXACTLY ONE of these, something is wrong. + else if (!bb_items.is_empty() && !next_items.is_empty()) || bb_items.len() > 1 { + let nts = bb_items + .iter() + .map(|item| match item.top_elts.get_tt(item.idx) { + TokenTree::MetaVarDecl(_, bind, name) => format!("{} ('{}')", name, bind), + _ => panic!(), + }) + .collect::>() + .join(" or "); + + return Error( + parser.span, + format!( + "local ambiguity: multiple parsing options: {}", + match next_items.len() { + 0 => format!("built-in NTs {}.", nts), + 1 => format!("built-in NTs {} or 1 other option.", nts), + n => format!("built-in NTs {} or {} other options.", nts, n), + } + ), + ); + } + // If there are no posible next positions AND we aren't waiting for the black-box parser, + // then their is a syntax error. + else if bb_items.is_empty() && next_items.is_empty() { return Failure(parser.span, parser.token); - } else if !next_items.is_empty() { - /* Now process the next token */ + } + // Dump all possible `next_items` into `cur_items` for the next iteration. + else if !next_items.is_empty() { + // Now process the next token cur_items.extend(next_items.drain(..)); parser.bump(); - } else /* bb_items.len() == 1 */ { + } + // Finally, we have the case where we need to call the black-box parser to get some + // nonterminal. + else { + assert_eq!(bb_items.len(), 1); + let mut item = bb_items.pop().unwrap(); if let TokenTree::MetaVarDecl(span, _, ident) = item.top_elts.get_tt(item.idx) { let match_cur = item.match_cur; - item.push_match(match_cur, - MatchedNonterminal(Rc::new(parse_nt(&mut parser, span, &ident.name.as_str())))); + item.push_match( + match_cur, + MatchedNonterminal(Rc::new(parse_nt(&mut parser, span, &ident.name.as_str()))), + ); item.idx += 1; item.match_cur += 1; } else { @@ -512,20 +721,21 @@ fn may_begin_with(name: &str, token: &Token) -> bool { "expr" => token.can_begin_expr(), "ty" => token.can_begin_type(), "ident" => token.is_ident(), - "vis" => match *token { // The follow-set of :vis + "priv" keyword + interpolated + "vis" => match *token { + // The follow-set of :vis + "priv" keyword + interpolated Token::Comma | Token::Ident(_) | Token::Interpolated(_) => true, _ => token.can_begin_type(), }, "block" => match *token { Token::OpenDelim(token::Brace) => true, Token::Interpolated(ref nt) => match nt.0 { - token::NtItem(_) | - token::NtPat(_) | - token::NtTy(_) | - token::NtIdent(_) | - token::NtMeta(_) | - token::NtPath(_) | - token::NtVis(_) => false, // none of these may start with '{'. + token::NtItem(_) + | token::NtPat(_) + | token::NtTy(_) + | token::NtIdent(_) + | token::NtMeta(_) + | token::NtPath(_) + | token::NtVis(_) => false, // none of these may start with '{'. _ => true, }, _ => false, @@ -562,6 +772,18 @@ fn may_begin_with(name: &str, token: &Token) -> bool { } } +/// A call to the "black-box" parser to parse some rust nonterminal. +/// +/// # Parameters +/// +/// - `p`: the "black-box" parser to use +/// - `sp`: the `Span` we want to parse +/// - `name`: the name of the metavar _matcher_ we want to match (e.g. `tt`, `ident`, `block`, +/// etc...) +/// +/// # Returns +/// +/// The parsed nonterminal. fn parse_nt<'a>(p: &mut Parser<'a>, sp: Span, name: &str) -> Nonterminal { if name == "tt" { return token::NtTT(p.parse_token_tree()); @@ -591,12 +813,15 @@ fn parse_nt<'a>(p: &mut Parser<'a>, sp: Span, name: &str) -> Nonterminal { "ident" => match p.token { token::Ident(sn) => { p.bump(); - token::NtIdent(Spanned::{node: sn, span: p.prev_span}) + token::NtIdent(Spanned:: { + node: sn, + span: p.prev_span, + }) } _ => { let token_str = pprust::token_to_string(&p.token); - p.fatal(&format!("expected ident, found {}", - &token_str[..])).emit(); + p.fatal(&format!("expected ident, found {}", &token_str[..])) + .emit(); FatalError.raise() } }, @@ -606,6 +831,6 @@ fn parse_nt<'a>(p: &mut Parser<'a>, sp: Span, name: &str) -> Nonterminal { "lifetime" => token::NtLifetime(p.expect_lifetime()), // this is not supposed to happen, since it has been checked // when compiling the macro. - _ => p.span_bug(sp, "invalid fragment specifier") + _ => p.span_bug(sp, "invalid fragment specifier"), } } diff --git a/src/libsyntax/ext/tt/quoted.rs b/src/libsyntax/ext/tt/quoted.rs index 0e21e3f6b0010..c55dfaba8f6b2 100644 --- a/src/libsyntax/ext/tt/quoted.rs +++ b/src/libsyntax/ext/tt/quoted.rs @@ -10,14 +10,16 @@ use ast; use ext::tt::macro_parser; -use parse::{ParseSess, token}; +use parse::{token, ParseSess}; use print::pprust; use symbol::keywords; -use syntax_pos::{DUMMY_SP, Span, BytePos}; +use syntax_pos::{BytePos, Span, DUMMY_SP}; use tokenstream; use std::rc::Rc; +/// Contains the sub-token-trees of a "delimited" token tree, such as the contents of `(`. Note +/// that the delimiter itself might be `NoDelim`. #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub struct Delimited { pub delim: token::DelimToken, @@ -25,14 +27,17 @@ pub struct Delimited { } impl Delimited { + /// Return the opening delimiter (possibly `NoDelim`). pub fn open_token(&self) -> token::Token { token::OpenDelim(self.delim) } + /// Return the closing delimiter (possibly `NoDelim`). pub fn close_token(&self) -> token::Token { token::CloseDelim(self.delim) } + /// Return a `self::TokenTree` with a `Span` corresponding to the opening delimiter. pub fn open_tt(&self, span: Span) -> TokenTree { let open_span = if span == DUMMY_SP { DUMMY_SP @@ -42,6 +47,7 @@ impl Delimited { TokenTree::Token(open_span, self.open_token()) } + /// Return a `self::TokenTree` with a `Span` corresponding to the closing delimiter. pub fn close_tt(&self, span: Span) -> TokenTree { let close_span = if span == DUMMY_SP { DUMMY_SP @@ -68,12 +74,14 @@ pub struct SequenceRepetition { /// for token sequences. #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)] pub enum KleeneOp { + /// Kleene star (`*`) for zero or more repetitions ZeroOrMore, + /// Kleene plus (`+`) for one or more repetitions OneOrMore, } /// Similar to `tokenstream::TokenTree`, except that `$i`, `$i:ident`, and `$(...)` -/// are "first-class" token trees. +/// are "first-class" token trees. Useful for parsing macros. #[derive(Debug, Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash)] pub enum TokenTree { Token(Span, token::Token), @@ -83,10 +91,15 @@ pub enum TokenTree { /// E.g. `$var` MetaVar(Span, ast::Ident), /// E.g. `$var:expr`. This is only used in the left hand side of MBE macros. - MetaVarDecl(Span, ast::Ident /* name to bind */, ast::Ident /* kind of nonterminal */), + MetaVarDecl( + Span, + ast::Ident, /* name to bind */ + ast::Ident, /* kind of nonterminal */ + ), } impl TokenTree { + /// Return the number of tokens in the tree. pub fn len(&self) -> usize { match *self { TokenTree::Delimited(_, ref delimed) => match delimed.delim { @@ -98,6 +111,8 @@ impl TokenTree { } } + /// Returns true if the given token tree contains no other tokens. This is vacuously true for + /// single tokens or metavar/decls, but may be false for delimited trees or sequences. pub fn is_empty(&self) -> bool { match *self { TokenTree::Delimited(_, ref delimed) => match delimed.delim { @@ -109,6 +124,7 @@ impl TokenTree { } } + /// Get the `index`-th sub-token-tree. This only makes sense for delimited trees and sequences. pub fn get_tt(&self, index: usize) -> TokenTree { match (self, index) { (&TokenTree::Delimited(_, ref delimed), _) if delimed.delim == token::NoDelim => { @@ -131,21 +147,48 @@ impl TokenTree { /// Retrieve the `TokenTree`'s span. pub fn span(&self) -> Span { match *self { - TokenTree::Token(sp, _) | - TokenTree::MetaVar(sp, _) | - TokenTree::MetaVarDecl(sp, _, _) | - TokenTree::Delimited(sp, _) | - TokenTree::Sequence(sp, _) => sp, + TokenTree::Token(sp, _) + | TokenTree::MetaVar(sp, _) + | TokenTree::MetaVarDecl(sp, _, _) + | TokenTree::Delimited(sp, _) + | TokenTree::Sequence(sp, _) => sp, } } } -pub fn parse(input: tokenstream::TokenStream, expect_matchers: bool, sess: &ParseSess) - -> Vec { +/// Takes a `tokenstream::TokenStream` and returns a `Vec`. Specifically, this +/// takes a generic `TokenStream`, such as is used in the rest of the compiler, and returns a +/// collection of `TokenTree` for use in parsing a macro. +/// +/// # Parameters +/// +/// - `input`: a token stream to read from, the contents of which we are parsing. +/// - `expect_matchers`: `parse` can be used to parse either the "patterns" or the "body" of a +/// macro. Both take roughly the same form _except_ that in a pattern, metavars are declared with +/// their "matcher" type. For example `$var:expr` or `$id:ident`. In this example, `expr` and +/// `ident` are "matchers". They are not present in the body of a macro rule -- just in the +/// pattern, so we pass a parameter to indicate whether to expect them or not. +/// - `sess`: the parsing session. Any errors will be emitted to this session. +/// +/// # Returns +/// +/// A collection of `self::TokenTree`. There may also be some errors emitted to `sess`. +pub fn parse( + input: tokenstream::TokenStream, + expect_matchers: bool, + sess: &ParseSess, +) -> Vec { + // Will contain the final collection of `self::TokenTree` let mut result = Vec::new(); + + // For each token tree in `input`, parse the token into a `self::TokenTree`, consuming + // additional trees if need be. let mut trees = input.trees(); while let Some(tree) = trees.next() { let tree = parse_tree(tree, &mut trees, expect_matchers, sess); + + // Given the parsed tree, if there is a metavar and we are expecting matchers, actually + // parse out the matcher (i.e. in `$id:ident` this would parse the `:` and `ident`). match tree { TokenTree::MetaVar(start_sp, ident) if expect_matchers => { let span = match trees.next() { @@ -154,78 +197,149 @@ pub fn parse(input: tokenstream::TokenStream, expect_matchers: bool, sess: &Pars Some(kind) => { let span = end_sp.with_lo(start_sp.lo()); result.push(TokenTree::MetaVarDecl(span, ident, kind)); - continue + continue; } _ => end_sp, }, - tree => tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(span), + tree => tree.as_ref() + .map(tokenstream::TokenTree::span) + .unwrap_or(span), }, - tree => tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(start_sp), + tree => tree.as_ref() + .map(tokenstream::TokenTree::span) + .unwrap_or(start_sp), }; sess.missing_fragment_specifiers.borrow_mut().insert(span); - result.push(TokenTree::MetaVarDecl(span, ident, keywords::Invalid.ident())); + result.push(TokenTree::MetaVarDecl( + span, + ident, + keywords::Invalid.ident(), + )); } + + // Not a metavar or no matchers allowed, so just return the tree _ => result.push(tree), } } result } -fn parse_tree(tree: tokenstream::TokenTree, - trees: &mut I, - expect_matchers: bool, - sess: &ParseSess) - -> TokenTree - where I: Iterator, +/// Takes a `tokenstream::TokenTree` and returns a `self::TokenTree`. Specifically, this takes a +/// generic `TokenTree`, such as is used in the rest of the compiler, and returns a `TokenTree` +/// for use in parsing a macro. +/// +/// Converting the given tree may involve reading more tokens. +/// +/// # Parameters +/// +/// - `tree`: the tree we wish to convert. +/// - `trees`: an iterator over trees. We may need to read more tokens from it in order to finish +/// converting `tree` +/// - `expect_matchers`: same as for `parse` (see above). +/// - `sess`: the parsing session. Any errors will be emitted to this session. +fn parse_tree( + tree: tokenstream::TokenTree, + trees: &mut I, + expect_matchers: bool, + sess: &ParseSess, +) -> TokenTree +where + I: Iterator, { + // Depending on what `tree` is, we could be parsing different parts of a macro match tree { + // `tree` is a `$` token. Look at the next token in `trees` tokenstream::TokenTree::Token(span, token::Dollar) => match trees.next() { + // `tree` is followed by a delimited set of token trees. This indicates the beginning + // of a repetition sequence in the macro (e.g. `$(pat)*`). Some(tokenstream::TokenTree::Delimited(span, delimited)) => { + // Must have `(` not `{` or `[` if delimited.delim != token::Paren { let tok = pprust::token_to_string(&token::OpenDelim(delimited.delim)); let msg = format!("expected `(`, found `{}`", tok); sess.span_diagnostic.span_err(span, &msg); } + // Parse the contents of the sequence itself let sequence = parse(delimited.tts.into(), expect_matchers, sess); + // Get the Kleene operator and optional separator let (separator, op) = parse_sep_and_kleene_op(trees, span, sess); + // Count the number of captured "names" (i.e. named metavars) let name_captures = macro_parser::count_names(&sequence); - TokenTree::Sequence(span, Rc::new(SequenceRepetition { - tts: sequence, - separator, - op, - num_captures: name_captures, - })) + TokenTree::Sequence( + span, + Rc::new(SequenceRepetition { + tts: sequence, + separator, + op, + num_captures: name_captures, + }), + ) } + + // `tree` is followed by an `ident`. This could be `$meta_var` or the `$crate` special + // metavariable that names the crate of the invokation. Some(tokenstream::TokenTree::Token(ident_span, ref token)) if token.is_ident() => { let ident = token.ident().unwrap(); let span = ident_span.with_lo(span.lo()); if ident.name == keywords::Crate.name() { - let ident = ast::Ident { name: keywords::DollarCrate.name(), ..ident }; + let ident = ast::Ident { + name: keywords::DollarCrate.name(), + ..ident + }; TokenTree::Token(span, token::Ident(ident)) } else { TokenTree::MetaVar(span, ident) } } + + // `tree` is followed by a random token. This is an error. Some(tokenstream::TokenTree::Token(span, tok)) => { - let msg = format!("expected identifier, found `{}`", pprust::token_to_string(&tok)); + let msg = format!( + "expected identifier, found `{}`", + pprust::token_to_string(&tok) + ); sess.span_diagnostic.span_err(span, &msg); TokenTree::MetaVar(span, keywords::Invalid.ident()) } + + // There are no more tokens. Just return the `$` we already have. None => TokenTree::Token(span, token::Dollar), }, + + // `tree` is an arbitrary token. Keep it. tokenstream::TokenTree::Token(span, tok) => TokenTree::Token(span, tok), - tokenstream::TokenTree::Delimited(span, delimited) => { - TokenTree::Delimited(span, Rc::new(Delimited { + + // `tree` is the beginning of a delimited set of tokens (e.g. `(` or `{`). We need to + // descend into the delimited set and further parse it. + tokenstream::TokenTree::Delimited(span, delimited) => TokenTree::Delimited( + span, + Rc::new(Delimited { delim: delimited.delim, tts: parse(delimited.tts.into(), expect_matchers, sess), - })) - } + }), + ), } } -fn parse_sep_and_kleene_op(input: &mut I, span: Span, sess: &ParseSess) - -> (Option, KleeneOp) - where I: Iterator, +/// Attempt to parse a single Kleene star, possibly with a separator. +/// +/// For example, in a pattern such as `$(a),*`, `a` is the pattern to be repeated, `,` is the +/// separator, and `*` is the Kleene operator. This function is specifically concerned with parsing +/// the last two tokens of such a pattern: namely, the optional separator and the Kleene operator +/// itself. Note that here we are parsing the _macro_ itself, rather than trying to match some +/// stream of tokens in an invocation of a macro. +/// +/// This function will take some input iterator `input` corresponding to `span` and a parsing +/// session `sess`. If the next one (or possibly two) tokens in `input` correspond to a Kleene +/// operator and separator, then a tuple with `(separator, KleeneOp)` is returned. Otherwise, an +/// error with the appropriate span is emitted to `sess` and a dummy value is returned. +fn parse_sep_and_kleene_op( + input: &mut I, + span: Span, + sess: &ParseSess, +) -> (Option, KleeneOp) +where + I: Iterator, { fn kleene_op(token: &token::Token) -> Option { match *token { @@ -235,20 +349,40 @@ fn parse_sep_and_kleene_op(input: &mut I, span: Span, sess: &ParseSess) } } + // We attempt to look at the next two token trees in `input`. I will call the first #1 and the + // second #2. If #1 and #2 don't match a valid KleeneOp with/without separator, that is an + // error, and we should emit an error on the most specific span possible. let span = match input.next() { + // #1 is a token Some(tokenstream::TokenTree::Token(span, tok)) => match kleene_op(&tok) { + // #1 is a KleeneOp with no separator Some(op) => return (None, op), + + // #1 is not a KleeneOp, but may be a separator... need to look at #2 None => match input.next() { + // #2 is a token Some(tokenstream::TokenTree::Token(span, tok2)) => match kleene_op(&tok2) { + // #2 is a KleeneOp, so #1 must be a separator Some(op) => return (Some(tok), op), + + // #2 is not a KleeneOp... error None => span, }, - tree => tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(span), - } + + // #2 is not a token at all... error + tree => tree.as_ref() + .map(tokenstream::TokenTree::span) + .unwrap_or(span), + }, }, - tree => tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(span), + + // #1 is not a token at all... error + tree => tree.as_ref() + .map(tokenstream::TokenTree::span) + .unwrap_or(span), }; + // Error... sess.span_diagnostic.span_err(span, "expected `*` or `+`"); (None, KleeneOp::ZeroOrMore) } diff --git a/src/test/run-pass/env-home-dir.rs b/src/test/run-pass/env-home-dir.rs index 22e440c6ffa51..9bbff1eeb81f3 100644 --- a/src/test/run-pass/env-home-dir.rs +++ b/src/test/run-pass/env-home-dir.rs @@ -27,7 +27,10 @@ fn main() { if cfg!(target_os = "android") { assert!(home_dir().is_none()); } else { - assert!(home_dir().is_some()); + // When HOME is not set, some platforms return `None`, + // but others return `Some` with a default. + // Just check that it is not "/home/MountainView". + assert_ne!(home_dir(), Some(PathBuf::from("/home/MountainView"))); } } diff --git a/src/test/run-pass/use-nested-groups.rs b/src/test/run-pass/use-nested-groups.rs index 74a82afd462b8..a28f8da9ff882 100644 --- a/src/test/run-pass/use-nested-groups.rs +++ b/src/test/run-pass/use-nested-groups.rs @@ -24,12 +24,19 @@ mod a { } } +// Test every possible part of the syntax use a::{B, d::{self, *, g::H}}; +// Test a more common use case +use std::sync::{Arc, atomic::{AtomicBool, Ordering}}; + fn main() { let _: B; let _: E; let _: F; let _: H; let _: d::g::I; + + let _: Arc; + let _: Ordering; } diff --git a/src/test/rustdoc/link-title-escape.rs b/src/test/rustdoc/link-title-escape.rs new file mode 100644 index 0000000000000..eb53c3c2cb52d --- /dev/null +++ b/src/test/rustdoc/link-title-escape.rs @@ -0,0 +1,19 @@ +// Copyright 2018 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +// compile-flags: -Z unstable-options --disable-commonmark + +#![crate_name = "foo"] + +//! hello [foo] +//! +//! [foo]: url 'title & & "things"' + +// @has 'foo/index.html' 'title & <stuff> & "things"' diff --git a/src/test/ui/cross-file-errors/main.rs b/src/test/ui/cross-file-errors/main.rs new file mode 100644 index 0000000000000..8eae79a21a983 --- /dev/null +++ b/src/test/ui/cross-file-errors/main.rs @@ -0,0 +1,16 @@ +// Copyright 2018 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +#[macro_use] +mod underscore; + +fn main() { + underscore!(); +} diff --git a/src/test/ui/cross-file-errors/main.stderr b/src/test/ui/cross-file-errors/main.stderr new file mode 100644 index 0000000000000..a1cdae10edfcd --- /dev/null +++ b/src/test/ui/cross-file-errors/main.stderr @@ -0,0 +1,11 @@ +error: expected expression, found `_` + --> $DIR/underscore.rs:18:9 + | +18 | _ + | ^ + | + ::: $DIR/main.rs:15:5 + | +15 | underscore!(); + | -------------- in this macro invocation + diff --git a/src/test/ui/cross-file-errors/underscore.rs b/src/test/ui/cross-file-errors/underscore.rs new file mode 100644 index 0000000000000..312b3b8f4ddd5 --- /dev/null +++ b/src/test/ui/cross-file-errors/underscore.rs @@ -0,0 +1,20 @@ +// Copyright 2018 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +// We want this file only so we can test cross-file error +// messages, but we don't want it in an external crate. +// ignore-test +#![crate_type = "lib"] + +macro_rules! underscore { + () => ( + _ + ) +} diff --git a/src/test/ui/macro_backtrace/main.stderr b/src/test/ui/macro_backtrace/main.stderr index 5990f71b3ca0a..48138ee711b3f 100644 --- a/src/test/ui/macro_backtrace/main.stderr +++ b/src/test/ui/macro_backtrace/main.stderr @@ -22,7 +22,7 @@ error: expected one of `!`, `.`, `::`, `;`, `?`, `{`, `}`, or an operator, found 27 | ping!(); | -------- in this macro invocation | - ::: + ::: :1:1 | 1 | ( ) => { pong ! ( ) ; } | ------------------------- @@ -42,7 +42,7 @@ error: expected one of `!`, `.`, `::`, `;`, `?`, `{`, `}`, or an operator, found 28 | deep!(); | -------- in this macro invocation (#1) | - ::: + ::: :1:1 | 1 | ( ) => { foo ! ( ) ; } | ------------------------ @@ -50,7 +50,7 @@ error: expected one of `!`, `.`, `::`, `;`, `?`, `{`, `}`, or an operator, found | | in this macro invocation (#2) | in this expansion of `deep!` (#1) | - ::: + ::: :1:1 | 1 | ( ) => { bar ! ( ) ; } | ------------------------ @@ -58,7 +58,7 @@ error: expected one of `!`, `.`, `::`, `;`, `?`, `{`, `}`, or an operator, found | | in this macro invocation (#3) | in this expansion of `foo!` (#2) | - ::: + ::: :1:1 | 1 | ( ) => { ping ! ( ) ; } | ------------------------- @@ -66,7 +66,7 @@ error: expected one of `!`, `.`, `::`, `;`, `?`, `{`, `}`, or an operator, found | | in this macro invocation (#4) | in this expansion of `bar!` (#3) | - ::: + ::: :1:1 | 1 | ( ) => { pong ! ( ) ; } | ------------------------- diff --git a/src/test/ui/use-nested-groups-error.rs b/src/test/ui/use-nested-groups-error.rs new file mode 100644 index 0000000000000..a9b6b3ee70d57 --- /dev/null +++ b/src/test/ui/use-nested-groups-error.rs @@ -0,0 +1,27 @@ +// Copyright 2018 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +#![feature(use_nested_groups)] + +mod a { + pub mod b1 { + pub enum C2 {} + } + + pub enum B2 {} +} + +use a::{b1::{C1, C2}, B2}; +//~^ ERROR unresolved import `a::b1::C1` + +fn main() { + let _: C2; + let _: B2; +} diff --git a/src/test/ui/use-nested-groups-error.stderr b/src/test/ui/use-nested-groups-error.stderr new file mode 100644 index 0000000000000..cae34684c8e38 --- /dev/null +++ b/src/test/ui/use-nested-groups-error.stderr @@ -0,0 +1,8 @@ +error[E0432]: unresolved import `a::b1::C1` + --> $DIR/use-nested-groups-error.rs:21:14 + | +21 | use a::{b1::{C1, C2}, B2}; + | ^^ no `C1` in `a::b1`. Did you mean to use `C2`? + +error: aborting due to previous error + diff --git a/src/tools/compiletest/src/runtest.rs b/src/tools/compiletest/src/runtest.rs index bf5fc00428df2..abf62a060b83b 100644 --- a/src/tools/compiletest/src/runtest.rs +++ b/src/tools/compiletest/src/runtest.rs @@ -1402,7 +1402,7 @@ impl<'test> TestCx<'test> { } /// For each `aux-build: foo/bar` annotation, we check to find the - /// file in a `aux` directory relative to the test itself. + /// file in a `auxiliary` directory relative to the test itself. fn compute_aux_test_paths(&self, rel_ab: &str) -> TestPaths { let test_ab = self.testpaths .file