Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

syntax: Remove Deref impl from Token #61669

Merged
merged 4 commits into from
Jun 9, 2019
Merged

syntax: Remove Deref impl from Token #61669

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
ffe2347
syntax: Keep full `Token`s for `macro_rules` separators
petrochenkov Jun 8, 2019
0ca3c2f
syntax: Move most of the `TokenKind` methods to `Token`
petrochenkov Jun 8, 2019
25b0514
syntax: Remove `Deref` impl from `Token`
petrochenkov Jun 8, 2019
9aaa7c7
syntax: Move some `Token` methods around
petrochenkov Jun 8, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/librustdoc/html/highlight.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -257,7 +257,7 @@ impl<'a> Classifier<'a> {
token::Question => Class::QuestionMark,

token::Dollar => {
if self.lexer.peek().kind.is_ident() {
if self.lexer.peek().is_ident() {
self.in_macro_nonterminal = true;
Class::MacroNonTerminal
} else {
Expand Down
5 changes: 2 additions & 3 deletions src/libsyntax/attr/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ use crate::source_map::{BytePos, Spanned, dummy_spanned};
use crate::parse::lexer::comments::{doc_comment_style, strip_doc_comment_decoration};
use crate::parse::parser::Parser;
use crate::parse::{self, ParseSess, PResult};
use crate::parse::token::{self, Token, TokenKind};
use crate::parse::token::{self, Token};
use crate::ptr::P;
use crate::symbol::{sym, Symbol};
use crate::ThinVec;
Expand Down Expand Up @@ -467,8 +467,7 @@ impl MetaItem {
segment.ident.span.ctxt());
idents.push(TokenTree::token(token::ModSep, mod_sep_span).into());
}
idents.push(TokenTree::token(TokenKind::from_ast_ident(segment.ident),
segment.ident.span).into());
idents.push(TokenTree::Token(Token::from_ast_ident(segment.ident)).into());
last_pos = segment.ident.span.hi();
}
self.node.tokens(self.span).append_to_tree_and_joint_vec(&mut idents);
Expand Down
28 changes: 14 additions & 14 deletions src/libsyntax/ext/tt/macro_parser.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ use crate::ast::{Ident, Name};
use crate::ext::tt::quoted::{self, TokenTree};
use crate::parse::{Directory, ParseSess};
use crate::parse::parser::{Parser, PathStyle};
use crate::parse::token::{self, DocComment, Nonterminal, Token, TokenKind};
use crate::parse::token::{self, DocComment, Nonterminal, Token};
use crate::print::pprust;
use crate::symbol::{kw, sym, Symbol};
use crate::tokenstream::{DelimSpan, TokenStream};
Expand Down Expand Up @@ -199,7 +199,7 @@ struct MatcherPos<'root, 'tt: 'root> {
seq_op: Option<quoted::KleeneOp>,

/// The separator if we are in a repetition.
sep: Option<TokenKind>,
sep: Option<Token>,

/// The "parent" matcher position if we are in a repetition. That is, the matcher position just
/// before we enter the sequence.
Expand Down Expand Up @@ -417,24 +417,24 @@ fn nameize<I: Iterator<Item = NamedMatch>>(

/// Generates an appropriate parsing failure message. For EOF, this is "unexpected end...". For
/// other tokens, this is "unexpected token...".
pub fn parse_failure_msg(tok: TokenKind) -> String {
match tok {
pub fn parse_failure_msg(tok: &Token) -> String {
match tok.kind {
token::Eof => "unexpected end of macro invocation".to_string(),
_ => format!(
"no rules expected the token `{}`",
pprust::token_to_string(&tok)
pprust::token_to_string(tok)
),
}
}

/// Performs a token equality check, ignoring syntax context (that is, an unhygienic comparison)
fn token_name_eq(t1: &TokenKind, t2: &TokenKind) -> bool {
if let (Some((name1, is_raw1)), Some((name2, is_raw2))) = (t1.ident_name(), t2.ident_name()) {
name1 == name2 && is_raw1 == is_raw2
} else if let (Some(name1), Some(name2)) = (t1.lifetime_name(), t2.lifetime_name()) {
name1 == name2
fn token_name_eq(t1: &Token, t2: &Token) -> bool {
if let (Some((ident1, is_raw1)), Some((ident2, is_raw2))) = (t1.ident(), t2.ident()) {
ident1.name == ident2.name && is_raw1 == is_raw2
} else if let (Some(ident1), Some(ident2)) = (t1.lifetime(), t2.lifetime()) {
ident1.name == ident2.name
} else {
*t1 == *t2
t1.kind == t2.kind
}
}

Expand Down Expand Up @@ -712,7 +712,7 @@ pub fn parse(

// If we reached the EOF, check that there is EXACTLY ONE possible matcher. Otherwise,
// either the parse is ambiguous (which should never happen) or there is a syntax error.
if token_name_eq(&parser.token, &token::Eof) {
if parser.token == token::Eof {
if eof_items.len() == 1 {
let matches = eof_items[0]
.matches
Expand Down Expand Up @@ -804,8 +804,8 @@ pub fn parse(

/// The token is an identifier, but not `_`.
/// We prohibit passing `_` to macros expecting `ident` for now.
fn get_macro_name(token: &TokenKind) -> Option<(Name, bool)> {
match *token {
fn get_macro_name(token: &Token) -> Option<(Name, bool)> {
match token.kind {
token::Ident(name, is_raw) if name != kw::Underscore => Some((name, is_raw)),
_ => None,
}
Expand Down
34 changes: 17 additions & 17 deletions src/libsyntax/ext/tt/macro_rules.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ use crate::symbol::{Symbol, kw, sym};
use crate::tokenstream::{DelimSpan, TokenStream, TokenTree};

use errors::FatalError;
use syntax_pos::{Span, DUMMY_SP, symbol::Ident};
use syntax_pos::{Span, symbol::Ident};
use log::debug;

use rustc_data_structures::fx::{FxHashMap};
Expand Down Expand Up @@ -200,7 +200,7 @@ fn generic_extension<'cx>(cx: &'cx mut ExtCtxt<'_>,

let (token, label) = best_failure.expect("ran no matchers");
let span = token.span.substitute_dummy(sp);
let mut err = cx.struct_span_err(span, &parse_failure_msg(token.kind));
let mut err = cx.struct_span_err(span, &parse_failure_msg(&token));
err.span_label(span, label);
if let Some(sp) = def_span {
if cx.source_map().span_to_filename(sp).is_real() && !sp.is_dummy() {
Expand Down Expand Up @@ -266,17 +266,19 @@ pub fn compile(
let argument_gram = vec![
quoted::TokenTree::Sequence(DelimSpan::dummy(), Lrc::new(quoted::SequenceRepetition {
tts: vec![
quoted::TokenTree::MetaVarDecl(DUMMY_SP, lhs_nm, ast::Ident::from_str("tt")),
quoted::TokenTree::token(token::FatArrow, DUMMY_SP),
quoted::TokenTree::MetaVarDecl(DUMMY_SP, rhs_nm, ast::Ident::from_str("tt")),
quoted::TokenTree::MetaVarDecl(def.span, lhs_nm, ast::Ident::from_str("tt")),
quoted::TokenTree::token(token::FatArrow, def.span),
quoted::TokenTree::MetaVarDecl(def.span, rhs_nm, ast::Ident::from_str("tt")),
],
separator: Some(if body.legacy { token::Semi } else { token::Comma }),
separator: Some(Token::new(
if body.legacy { token::Semi } else { token::Comma }, def.span
)),
op: quoted::KleeneOp::OneOrMore,
num_captures: 2,
})),
// to phase into semicolon-termination instead of semicolon-separation
quoted::TokenTree::Sequence(DelimSpan::dummy(), Lrc::new(quoted::SequenceRepetition {
tts: vec![quoted::TokenTree::token(token::Semi, DUMMY_SP)],
tts: vec![quoted::TokenTree::token(token::Semi, def.span)],
separator: None,
op: quoted::KleeneOp::ZeroOrMore,
num_captures: 0
Expand All @@ -286,7 +288,7 @@ pub fn compile(
let argument_map = match parse(sess, body.stream(), &argument_gram, None, true) {
Success(m) => m,
Failure(token, msg) => {
let s = parse_failure_msg(token.kind);
let s = parse_failure_msg(&token);
let sp = token.span.substitute_dummy(def.span);
let mut err = sess.span_diagnostic.struct_span_fatal(sp, &s);
err.span_label(sp, msg);
Expand Down Expand Up @@ -608,9 +610,8 @@ impl FirstSets {
// If the sequence contents can be empty, then the first
// token could be the separator token itself.

if let (Some(ref sep), true) = (seq_rep.separator.clone(),
subfirst.maybe_empty) {
first.add_one_maybe(TokenTree::token(sep.clone(), sp.entire()));
if let (Some(sep), true) = (&seq_rep.separator, subfirst.maybe_empty) {
first.add_one_maybe(TokenTree::Token(sep.clone()));
}

// Reverse scan: Sequence comes before `first`.
Expand Down Expand Up @@ -658,9 +659,8 @@ impl FirstSets {
// If the sequence contents can be empty, then the first
// token could be the separator token itself.

if let (Some(ref sep), true) = (seq_rep.separator.clone(),
subfirst.maybe_empty) {
first.add_one_maybe(TokenTree::token(sep.clone(), sp.entire()));
if let (Some(sep), true) = (&seq_rep.separator, subfirst.maybe_empty) {
first.add_one_maybe(TokenTree::Token(sep.clone()));
}

assert!(first.maybe_empty);
Expand Down Expand Up @@ -851,7 +851,7 @@ fn check_matcher_core(sess: &ParseSess,
// against SUFFIX
continue 'each_token;
}
TokenTree::Sequence(sp, ref seq_rep) => {
TokenTree::Sequence(_, ref seq_rep) => {
suffix_first = build_suffix_first();
// The trick here: when we check the interior, we want
// to include the separator (if any) as a potential
Expand All @@ -864,9 +864,9 @@ fn check_matcher_core(sess: &ParseSess,
// work of cloning it? But then again, this way I may
// get a "tighter" span?
let mut new;
let my_suffix = if let Some(ref u) = seq_rep.separator {
let my_suffix = if let Some(sep) = &seq_rep.separator {
new = suffix_first.clone();
new.add_one_maybe(TokenTree::token(u.clone(), sp.entire()));
new.add_one_maybe(TokenTree::Token(sep.clone()));
&new
} else {
&suffix_first
Expand Down
34 changes: 12 additions & 22 deletions src/libsyntax/ext/tt/quoted.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,24 +23,14 @@ pub struct Delimited {
}

impl Delimited {
/// Returns the opening delimiter (possibly `NoDelim`).
pub fn open_token(&self) -> TokenKind {
token::OpenDelim(self.delim)
}

/// Returns the closing delimiter (possibly `NoDelim`).
pub fn close_token(&self) -> TokenKind {
token::CloseDelim(self.delim)
}

/// Returns a `self::TokenTree` with a `Span` corresponding to the opening delimiter.
pub fn open_tt(&self, span: Span) -> TokenTree {
let open_span = if span.is_dummy() {
span
} else {
span.with_lo(span.lo() + BytePos(self.delim.len() as u32))
};
TokenTree::token(self.open_token(), open_span)
TokenTree::token(token::OpenDelim(self.delim), open_span)
}

/// Returns a `self::TokenTree` with a `Span` corresponding to the closing delimiter.
Expand All @@ -50,7 +40,7 @@ impl Delimited {
} else {
span.with_lo(span.hi() - BytePos(self.delim.len() as u32))
};
TokenTree::token(self.close_token(), close_span)
TokenTree::token(token::CloseDelim(self.delim), close_span)
}
}

Expand All @@ -59,7 +49,7 @@ pub struct SequenceRepetition {
/// The sequence of token trees
pub tts: Vec<TokenTree>,
/// The optional separator
pub separator: Option<TokenKind>,
pub separator: Option<Token>,
/// Whether the sequence can be repeated zero (*), or one or more times (+)
pub op: KleeneOp,
/// The number of `Match`s that appear in the sequence (and subsequences)
Expand Down Expand Up @@ -282,7 +272,7 @@ where
Some(tokenstream::TokenTree::Delimited(span, delim, tts)) => {
// Must have `(` not `{` or `[`
if delim != token::Paren {
let tok = pprust::token_to_string(&token::OpenDelim(delim));
let tok = pprust::token_kind_to_string(&token::OpenDelim(delim));
let msg = format!("expected `(`, found `{}`", tok);
sess.span_diagnostic.span_err(span.entire(), &msg);
}
Expand Down Expand Up @@ -371,8 +361,8 @@ where

/// Takes a token and returns `Some(KleeneOp)` if the token is `+` `*` or `?`. Otherwise, return
/// `None`.
fn kleene_op(token: &TokenKind) -> Option<KleeneOp> {
match *token {
fn kleene_op(token: &Token) -> Option<KleeneOp> {
match token.kind {
token::BinOp(token::Star) => Some(KleeneOp::ZeroOrMore),
token::BinOp(token::Plus) => Some(KleeneOp::OneOrMore),
token::Question => Some(KleeneOp::ZeroOrOne),
Expand Down Expand Up @@ -424,7 +414,7 @@ fn parse_sep_and_kleene_op<I>(
attrs: &[ast::Attribute],
edition: Edition,
macro_node_id: NodeId,
) -> (Option<TokenKind>, KleeneOp)
) -> (Option<Token>, KleeneOp)
where
I: Iterator<Item = tokenstream::TokenTree>,
{
Expand All @@ -449,7 +439,7 @@ fn parse_sep_and_kleene_op_2015<I>(
_features: &Features,
_attrs: &[ast::Attribute],
macro_node_id: NodeId,
) -> (Option<TokenKind>, KleeneOp)
) -> (Option<Token>, KleeneOp)
where
I: Iterator<Item = tokenstream::TokenTree>,
{
Expand Down Expand Up @@ -502,7 +492,7 @@ where
a hard error in an upcoming edition",
);

return (Some(token::Question), op);
return (Some(Token::new(token::Question, op1_span)), op);
}

// #2 is a random token (this is an error) :(
Expand Down Expand Up @@ -541,7 +531,7 @@ where
}

// #2 is a KleeneOp :D
Ok(Ok((op, _))) => return (Some(token.kind), op),
Ok(Ok((op, _))) => return (Some(token), op),

// #2 is a random token :(
Ok(Err(token)) => token.span,
Expand All @@ -567,7 +557,7 @@ fn parse_sep_and_kleene_op_2018<I>(
sess: &ParseSess,
_features: &Features,
_attrs: &[ast::Attribute],
) -> (Option<TokenKind>, KleeneOp)
) -> (Option<Token>, KleeneOp)
where
I: Iterator<Item = tokenstream::TokenTree>,
{
Expand Down Expand Up @@ -596,7 +586,7 @@ where
}

// #2 is a KleeneOp :D
Ok(Ok((op, _))) => return (Some(token.kind), op),
Ok(Ok((op, _))) => return (Some(token), op),

// #2 is a random token :(
Ok(Err(token)) => token.span,
Expand Down
21 changes: 8 additions & 13 deletions src/libsyntax/ext/tt/transcribe.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,10 @@ use crate::ext::expand::Marker;
use crate::ext::tt::macro_parser::{MatchedNonterminal, MatchedSeq, NamedMatch};
use crate::ext::tt::quoted;
use crate::mut_visit::noop_visit_tt;
use crate::parse::token::{self, NtTT, TokenKind};
use crate::parse::token::{self, NtTT, Token};
use crate::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndJoint};

use smallvec::{smallvec, SmallVec};
use syntax_pos::DUMMY_SP;

use rustc_data_structures::fx::FxHashMap;
use rustc_data_structures::sync::Lrc;
Expand All @@ -18,7 +17,7 @@ use std::rc::Rc;
/// An iterator over the token trees in a delimited token tree (`{ ... }`) or a sequence (`$(...)`).
enum Frame {
Delimited { forest: Lrc<quoted::Delimited>, idx: usize, span: DelimSpan },
Sequence { forest: Lrc<quoted::SequenceRepetition>, idx: usize, sep: Option<TokenKind> },
Sequence { forest: Lrc<quoted::SequenceRepetition>, idx: usize, sep: Option<Token> },
}

impl Frame {
Expand Down Expand Up @@ -109,17 +108,13 @@ pub fn transcribe(
else {
// Otherwise, if we have just reached the end of a sequence and we can keep repeating,
// go back to the beginning of the sequence.
if let Frame::Sequence { ref mut idx, ref sep, .. } = *stack.last_mut().unwrap() {
let (ref mut repeat_idx, repeat_len) = *repeats.last_mut().unwrap();
if let Frame::Sequence { idx, sep, .. } = stack.last_mut().unwrap() {
let (repeat_idx, repeat_len) = repeats.last_mut().unwrap();
*repeat_idx += 1;
if *repeat_idx < repeat_len {
if repeat_idx < repeat_len {
*idx = 0;
if let Some(sep) = sep.clone() {
let prev_span = match result.last() {
Some((tt, _)) => tt.span(),
None => DUMMY_SP,
};
result.push(TokenTree::token(sep, prev_span).into());
if let Some(sep) = sep {
result.push(TokenTree::Token(sep.clone()).into());
}
continue;
}
Expand Down Expand Up @@ -242,7 +237,7 @@ pub fn transcribe(
Ident::new(ident.name, ident.span.apply_mark(cx.current_expansion.mark));
sp = sp.apply_mark(cx.current_expansion.mark);
result.push(TokenTree::token(token::Dollar, sp).into());
result.push(TokenTree::token(TokenKind::from_ast_ident(ident), sp).into());
result.push(TokenTree::Token(Token::from_ast_ident(ident)).into());
}
}

Expand Down
2 changes: 1 addition & 1 deletion src/libsyntax/parse/diagnostics.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -729,7 +729,7 @@ impl<'a> Parser<'a> {
&mut self,
t: &TokenKind,
) -> PResult<'a, bool /* recovered */> {
let token_str = pprust::token_to_string(t);
let token_str = pprust::token_kind_to_string(t);
let this_token_str = self.this_token_descr();
let (prev_sp, sp) = match (&self.token.kind, self.subparser_name) {
// Point at the end of the macro call when reaching end of macro arguments.
Expand Down
Loading