From ed76f84cc80d63f6cb47dab67d0d989b3f076d7e Mon Sep 17 00:00:00 2001 From: mejrs <59372212+mejrs@users.noreply.github.com> Date: Sat, 25 Jul 2026 15:15:59 +0200 Subject: [PATCH 1/2] remove `hidden_glob_reexports` --- compiler/rustc_ast/src/token.rs | 33 +++++++++++++++------------------ 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/compiler/rustc_ast/src/token.rs b/compiler/rustc_ast/src/token.rs index f603e8604e646..efa2a008e65ae 100644 --- a/compiler/rustc_ast/src/token.rs +++ b/compiler/rustc_ast/src/token.rs @@ -8,10 +8,7 @@ pub use TokenKind::*; use rustc_macros::{Decodable, Encodable, StableHash}; use rustc_span::edition::Edition; use rustc_span::symbol::IdentPrintMode; -use rustc_span::{DUMMY_SP, ErrorGuaranteed, Span, kw, sym}; -#[allow(clippy::useless_attribute)] // FIXME: following use of `hidden_glob_reexports` incorrectly triggers `useless_attribute` lint. -#[allow(hidden_glob_reexports)] -use rustc_span::{Ident, Symbol}; +use rustc_span::{self as sp, DUMMY_SP, ErrorGuaranteed, Span, Symbol, kw, sym}; use crate::ast; use crate::util::case::Case; @@ -506,7 +503,7 @@ pub enum TokenKind { /// This identifier (and its span) is the identifier passed to the /// declarative macro. The span in the surrounding `Token` is the span of /// the `ident` metavariable in the macro's RHS. - NtIdent(Ident, IdentIsRaw), + NtIdent(sp::Ident, IdentIsRaw), /// Lifetime identifier token. /// Do not forget about `NtLifetime` when you want to match on lifetime identifiers. @@ -517,7 +514,7 @@ pub enum TokenKind { /// This identifier (and its span) is the lifetime passed to the /// declarative macro. The span in the surrounding `Token` is the span of /// the `lifetime` metavariable in the macro's RHS. - NtLifetime(Ident, IdentIsRaw), + NtLifetime(sp::Ident, IdentIsRaw), /// A doc comment token. /// `Symbol` is the doc comment's data excluding its "quotes" (`///`, `/**`, etc) @@ -637,7 +634,7 @@ impl Token { } /// Recovers a `Token` from an `Ident`. This creates a raw identifier if necessary. - pub fn from_ast_ident(ident: Ident) -> Self { + pub fn from_ast_ident(ident: sp::Ident) -> Self { Token::new(Ident(ident.name, ident.is_raw_guess().into()), ident.span) } @@ -845,10 +842,10 @@ impl Token { /// Returns an identifier if this token is an identifier. #[inline] - pub fn ident(&self) -> Option<(Ident, IdentIsRaw)> { + pub fn ident(&self) -> Option<(sp::Ident, IdentIsRaw)> { // We avoid using `Token::uninterpolate` here because it's slow. match self.kind { - Ident(name, is_raw) => Some((Ident::new(name, self.span), is_raw)), + Ident(name, is_raw) => Some((sp::Ident::new(name, self.span), is_raw)), NtIdent(ident, is_raw) => Some((ident, is_raw)), _ => None, } @@ -856,10 +853,10 @@ impl Token { /// Returns a lifetime identifier if this token is a lifetime. #[inline] - pub fn lifetime(&self) -> Option<(Ident, IdentIsRaw)> { + pub fn lifetime(&self) -> Option<(sp::Ident, IdentIsRaw)> { // We avoid using `Token::uninterpolate` here because it's slow. match self.kind { - Lifetime(name, is_raw) => Some((Ident::new(name, self.span), is_raw)), + Lifetime(name, is_raw) => Some((sp::Ident::new(name, self.span), is_raw)), NtLifetime(ident, is_raw) => Some((ident, is_raw)), _ => None, } @@ -934,32 +931,32 @@ impl Token { } pub fn is_path_segment_keyword(&self) -> bool { - self.is_non_raw_ident_where(Ident::is_path_segment_keyword) + self.is_non_raw_ident_where(sp::Ident::is_path_segment_keyword) } /// Returns true for reserved identifiers used internally for elided lifetimes, /// unnamed method parameters, crate root module, error recovery etc. pub fn is_special_ident(&self) -> bool { - self.is_non_raw_ident_where(Ident::is_special) + self.is_non_raw_ident_where(sp::Ident::is_special) } /// Returns `true` if the token is a keyword used in the language. pub fn is_used_keyword(&self) -> bool { - self.is_non_raw_ident_where(Ident::is_used_keyword) + self.is_non_raw_ident_where(sp::Ident::is_used_keyword) } /// Returns `true` if the token is a keyword reserved for possible future use. pub fn is_unused_keyword(&self) -> bool { - self.is_non_raw_ident_where(Ident::is_unused_keyword) + self.is_non_raw_ident_where(sp::Ident::is_unused_keyword) } /// Returns `true` if the token is either a special identifier or a keyword. pub fn is_reserved_ident(&self) -> bool { - self.is_non_raw_ident_where(Ident::is_reserved) + self.is_non_raw_ident_where(sp::Ident::is_reserved) } pub fn is_non_reserved_ident(&self) -> bool { - self.ident().is_some_and(|(id, raw)| raw == IdentIsRaw::Yes || !Ident::is_reserved(id)) + self.ident().is_some_and(|(id, raw)| raw == IdentIsRaw::Yes || !sp::Ident::is_reserved(id)) } /// Returns `true` if the token is the identifier `true` or `false`. @@ -980,7 +977,7 @@ impl Token { } /// Returns `true` if the token is a non-raw identifier for which `pred` holds. - pub fn is_non_raw_ident_where(&self, pred: impl FnOnce(Ident) -> bool) -> bool { + pub fn is_non_raw_ident_where(&self, pred: impl FnOnce(sp::Ident) -> bool) -> bool { match self.ident() { Some((id, IdentIsRaw::No)) => pred(id), _ => false, From f5b4f8d2091c4e9e7a30f8be10dfd333e019fbc2 Mon Sep 17 00:00:00 2001 From: mejrs <59372212+mejrs@users.noreply.github.com> Date: Sat, 25 Jul 2026 16:43:11 +0200 Subject: [PATCH 2/2] Prefix `rustc_ast::token` types with `tk::` --- compiler/rustc_ast_pretty/src/pprust/state.rs | 229 ++++++------ .../rustc_expand/src/proc_macro_server.rs | 333 +++++++++--------- 2 files changed, 283 insertions(+), 279 deletions(-) diff --git a/compiler/rustc_ast_pretty/src/pprust/state.rs b/compiler/rustc_ast_pretty/src/pprust/state.rs index 3b0c90264e32d..64c0be27a2daa 100644 --- a/compiler/rustc_ast_pretty/src/pprust/state.rs +++ b/compiler/rustc_ast_pretty/src/pprust/state.rs @@ -10,14 +10,13 @@ use std::borrow::Cow; use std::sync::Arc; use rustc_ast::attr::AttrIdGenerator; -use rustc_ast::token::{self, CommentKind, Delimiter, DocFragmentKind, Token, TokenKind}; use rustc_ast::tokenstream::{Spacing, TokenStream, TokenTree}; use rustc_ast::util::classify; use rustc_ast::util::comments::{Comment, CommentStyle}; use rustc_ast::{ self as ast, AttrArgs, AttrKind, BindingMode, BlockCheckMode, ByRef, DelimArgs, GenericArg, GenericBound, InlineAsmOperand, InlineAsmOptions, InlineAsmRegOrRegClass, - InlineAsmTemplatePiece, PatKind, RangeEnd, RangeSyntax, SelfKind, Term, attr, + InlineAsmTemplatePiece, PatKind, RangeEnd, RangeSyntax, SelfKind, Term, attr, token as tk, }; use rustc_span::edition::Edition; use rustc_span::source_map::SourceMap; @@ -327,18 +326,14 @@ fn print_crate_inner<'a>( /// E.g. `ident` + `where` would merge into `identwhere`. fn idents_would_merge(tt1: &TokenTree, tt2: &TokenTree) -> bool { fn is_ident_like(tt: &TokenTree) -> bool { - matches!( - tt, - TokenTree::Token(Token { kind: token::Ident(..) | token::NtIdent(..), .. }, _,) - ) + matches!(tt, TokenTree::Token(tk::Token { kind: tk::Ident(..) | tk::NtIdent(..), .. }, _,)) } is_ident_like(tt1) && is_ident_like(tt2) } fn space_between(tt1: &TokenTree, tt2: &TokenTree) -> bool { - use Delimiter::*; use TokenTree::{Delimited as Del, Token as Tok}; - use token::*; + use tk::Delimiter::{Bracket, Parenthesis}; fn is_punct(tt: &TokenTree) -> bool { matches!(tt, TokenTree::Token(tok, _) if tok.is_punct()) @@ -349,58 +344,64 @@ fn space_between(tt1: &TokenTree, tt2: &TokenTree) -> bool { // this match. match (tt1, tt2) { // No space after line doc comments. - (Tok(Token { kind: DocComment(CommentKind::Line, ..), .. }, _), _) => false, + (Tok(tk::Token { kind: tk::DocComment(tk::CommentKind::Line, ..), .. }, _), _) => false, // `.` + NON-PUNCT: `x.y`, `tup.0` - (Tok(Token { kind: Dot, .. }, _), tt2) if !is_punct(tt2) => false, + (Tok(tk::Token { kind: tk::Dot, .. }, _), tt2) if !is_punct(tt2) => false, // `$` + IDENT: `$e` - (Tok(Token { kind: Dollar, .. }, _), Tok(Token { kind: Ident(..), .. }, _)) => false, + ( + Tok(tk::Token { kind: tk::Dollar, .. }, _), + Tok(tk::Token { kind: tk::Ident(..), .. }, _), + ) => false, // NON-PUNCT + `,`: `foo,` // NON-PUNCT + `;`: `x = 3;`, `[T; 3]` // NON-PUNCT + `.`: `x.y`, `tup.0` - (tt1, Tok(Token { kind: Comma | Semi | Dot, .. }, _)) if !is_punct(tt1) => false, + (tt1, Tok(tk::Token { kind: tk::Comma | tk::Semi | tk::Dot, .. }, _)) if !is_punct(tt1) => { + false + } // IDENT + `!`: `println!()`, but `if !x { ... }` needs a space after the `if` - (Tok(Token { kind: Ident(sym, is_raw), span }, _), Tok(Token { kind: Bang, .. }, _)) - if !Ident::new(*sym, *span).is_reserved() || matches!(is_raw, IdentIsRaw::Yes) => - { + ( + Tok(tk::Token { kind: tk::Ident(sym, is_raw), span }, _), + Tok(tk::Token { kind: tk::Bang, .. }, _), + ) if !Ident::new(*sym, *span).is_reserved() || matches!(is_raw, tk::IdentIsRaw::Yes) => { false } // IDENT|`fn`|`Self`|`pub` + `(`: `f(3)`, `fn(x: u8)`, `Self()`, `pub(crate)`, // but `let (a, b) = (1, 2)` needs a space after the `let` - (Tok(Token { kind: Ident(sym, is_raw), span }, _), Del(_, _, Parenthesis, _)) + (Tok(tk::Token { kind: tk::Ident(sym, is_raw), span }, _), Del(_, _, Parenthesis, _)) if !Ident::new(*sym, *span).is_reserved() || *sym == kw::Fn || *sym == kw::SelfUpper || *sym == kw::Pub - || matches!(is_raw, IdentIsRaw::Yes) => + || matches!(is_raw, tk::IdentIsRaw::Yes) => { false } // `#` + `[`: `#[attr]` - (Tok(Token { kind: Pound, .. }, _), Del(_, _, Bracket, _)) => false, + (Tok(tk::Token { kind: tk::Pound, .. }, _), Del(_, _, Bracket, _)) => false, _ => true, } } pub fn doc_comment_to_string( - fragment_kind: DocFragmentKind, + fragment_kind: tk::DocFragmentKind, attr_style: ast::AttrStyle, data: Symbol, ) -> String { match fragment_kind { - DocFragmentKind::Sugared(comment_kind) => match (comment_kind, attr_style) { - (CommentKind::Line, ast::AttrStyle::Outer) => format!("///{data}"), - (CommentKind::Line, ast::AttrStyle::Inner) => format!("//!{data}"), - (CommentKind::Block, ast::AttrStyle::Outer) => format!("/**{data}*/"), - (CommentKind::Block, ast::AttrStyle::Inner) => format!("/*!{data}*/"), + tk::DocFragmentKind::Sugared(comment_kind) => match (comment_kind, attr_style) { + (tk::CommentKind::Line, ast::AttrStyle::Outer) => format!("///{data}"), + (tk::CommentKind::Line, ast::AttrStyle::Inner) => format!("//!{data}"), + (tk::CommentKind::Block, ast::AttrStyle::Outer) => format!("/**{data}*/"), + (tk::CommentKind::Block, ast::AttrStyle::Inner) => format!("/*!{data}*/"), }, - DocFragmentKind::Raw(_) => { + tk::DocFragmentKind::Raw(_) => { format!( "#{}[doc = {:?}]", if attr_style == ast::AttrStyle::Inner { "!" } else { "" }, @@ -410,24 +411,24 @@ pub fn doc_comment_to_string( } } -fn literal_to_string(lit: token::Lit) -> String { - let token::Lit { kind, symbol, suffix } = lit; +fn literal_to_string(lit: tk::Lit) -> String { + let tk::Lit { kind, symbol, suffix } = lit; let mut out = match kind { - token::Byte => format!("b'{symbol}'"), - token::Char => format!("'{symbol}'"), - token::Str => format!("\"{symbol}\""), - token::StrRaw(n) => { + tk::Byte => format!("b'{symbol}'"), + tk::Char => format!("'{symbol}'"), + tk::Str => format!("\"{symbol}\""), + tk::StrRaw(n) => { format!("r{delim}\"{string}\"{delim}", delim = "#".repeat(n as usize), string = symbol) } - token::ByteStr => format!("b\"{symbol}\""), - token::ByteStrRaw(n) => { + tk::ByteStr => format!("b\"{symbol}\""), + tk::ByteStrRaw(n) => { format!("br{delim}\"{string}\"{delim}", delim = "#".repeat(n as usize), string = symbol) } - token::CStr => format!("c\"{symbol}\""), - token::CStrRaw(n) => { + tk::CStr => format!("c\"{symbol}\""), + tk::CStrRaw(n) => { format!("cr{delim}\"{symbol}\"{delim}", delim = "#".repeat(n as usize)) } - token::Integer | token::Float | token::Bool | token::Err(_) => symbol.to_string(), + tk::Integer | tk::Float | tk::Bool | tk::Err(_) => symbol.to_string(), }; if let Some(suffix) = suffix { @@ -688,7 +689,7 @@ pub trait PrintState<'a>: std::ops::Deref + std::ops::Dere ast::AttrKind::Synthetic(..) => unreachable!(), // due to early return above ast::AttrKind::DocComment(comment_kind, data) => { self.word(doc_comment_to_string( - DocFragmentKind::Sugared(*comment_kind), + tk::DocFragmentKind::Sugared(*comment_kind), attr.style, *data, )); @@ -751,21 +752,21 @@ pub trait PrintState<'a>: std::ops::Deref + std::ops::Dere // Emit hygiene annotations for identity-bearing tokens, // matching how print_ident() and print_lifetime() call ann_post(). match token.kind { - token::Ident(name, _) => { + tk::Ident(name, _) => { self.ann_post(Ident::new(name, token.span)); } - token::NtIdent(ident, _) => { + tk::NtIdent(ident, _) => { self.ann_post(ident); } - token::Lifetime(name, _) => { + tk::Lifetime(name, _) => { self.ann_post(Ident::new(name, token.span)); } - token::NtLifetime(ident, _) => { + tk::NtLifetime(ident, _) => { self.ann_post(ident); } _ => {} } - if let token::DocComment(..) = token.kind { + if let tk::DocComment(..) = token.kind { self.hardbreak() } *spacing @@ -807,7 +808,7 @@ pub trait PrintState<'a>: std::ops::Deref + std::ops::Dere // `,` is better printed as `x,` than `x ,`. (Even if the original source // code was `x ,`.) // - // Finally, we must be careful about changing the output. Token pretty + // Finally, we must be careful about changing the output. tk::Token pretty // printing is used by `stringify!` and `impl Display for // proc_macro::TokenStream`, and some programs rely on the output having a // particular form, even though they shouldn't. In particular, some proc @@ -839,13 +840,13 @@ pub trait PrintState<'a>: std::ops::Deref + std::ops::Dere header: Option>, has_bang: bool, ident: Option, - delim: Delimiter, + delim: tk::Delimiter, open_spacing: Option, tts: &TokenStream, convert_dollar_crate: bool, span: Span, ) { - let cb = (delim == Delimiter::Brace).then(|| self.cbox(INDENT_UNIT)); + let cb = (delim == tk::Delimiter::Brace).then(|| self.cbox(INDENT_UNIT)); match header { Some(MacHeader::Path(path)) => self.print_path(path, false, 0), Some(MacHeader::Keyword(kw)) => self.word(kw), @@ -859,7 +860,7 @@ pub trait PrintState<'a>: std::ops::Deref + std::ops::Dere self.print_ident(ident); } match delim { - Delimiter::Brace => { + tk::Delimiter::Brace => { if header.is_some() || has_bang || ident.is_some() { self.nbsp(); } @@ -1004,105 +1005,109 @@ pub trait PrintState<'a>: std::ops::Deref + std::ops::Dere } /// Print the token kind precisely, without converting `$crate` into its respective crate name. - fn token_kind_to_string(&self, tok: &TokenKind) -> Cow<'static, str> { + fn token_kind_to_string(&self, tok: &tk::TokenKind) -> Cow<'static, str> { self.token_kind_to_string_ext(tok, None) } fn token_kind_to_string_ext( &self, - tok: &TokenKind, + tok: &tk::TokenKind, convert_dollar_crate: Option, ) -> Cow<'static, str> { match *tok { - token::Eq => "=".into(), - token::Lt => "<".into(), - token::Le => "<=".into(), - token::EqEq => "==".into(), - token::Ne => "!=".into(), - token::Ge => ">=".into(), - token::Gt => ">".into(), - token::Bang => "!".into(), - token::Tilde => "~".into(), - token::OrOr => "||".into(), - token::AndAnd => "&&".into(), - token::Plus => "+".into(), - token::Minus => "-".into(), - token::Star => "*".into(), - token::Slash => "/".into(), - token::Percent => "%".into(), - token::Caret => "^".into(), - token::And => "&".into(), - token::Or => "|".into(), - token::Shl => "<<".into(), - token::Shr => ">>".into(), - token::PlusEq => "+=".into(), - token::MinusEq => "-=".into(), - token::StarEq => "*=".into(), - token::SlashEq => "/=".into(), - token::PercentEq => "%=".into(), - token::CaretEq => "^=".into(), - token::AndEq => "&=".into(), - token::OrEq => "|=".into(), - token::ShlEq => "<<=".into(), - token::ShrEq => ">>=".into(), + tk::Eq => "=".into(), + tk::Lt => "<".into(), + tk::Le => "<=".into(), + tk::EqEq => "==".into(), + tk::Ne => "!=".into(), + tk::Ge => ">=".into(), + tk::Gt => ">".into(), + tk::Bang => "!".into(), + tk::Tilde => "~".into(), + tk::OrOr => "||".into(), + tk::AndAnd => "&&".into(), + tk::Plus => "+".into(), + tk::Minus => "-".into(), + tk::Star => "*".into(), + tk::Slash => "/".into(), + tk::Percent => "%".into(), + tk::Caret => "^".into(), + tk::And => "&".into(), + tk::Or => "|".into(), + tk::Shl => "<<".into(), + tk::Shr => ">>".into(), + tk::PlusEq => "+=".into(), + tk::MinusEq => "-=".into(), + tk::StarEq => "*=".into(), + tk::SlashEq => "/=".into(), + tk::PercentEq => "%=".into(), + tk::CaretEq => "^=".into(), + tk::AndEq => "&=".into(), + tk::OrEq => "|=".into(), + tk::ShlEq => "<<=".into(), + tk::ShrEq => ">>=".into(), /* Structural symbols */ - token::At => "@".into(), - token::Dot => ".".into(), - token::DotDot => "..".into(), - token::DotDotDot => "...".into(), - token::DotDotEq => "..=".into(), - token::Comma => ",".into(), - token::Semi => ";".into(), - token::Colon => ":".into(), - token::PathSep => "::".into(), - token::RArrow => "->".into(), - token::LArrow => "<-".into(), - token::FatArrow => "=>".into(), - token::OpenParen => "(".into(), - token::CloseParen => ")".into(), - token::OpenBracket => "[".into(), - token::CloseBracket => "]".into(), - token::OpenBrace => "{".into(), - token::CloseBrace => "}".into(), - token::OpenInvisible(_) | token::CloseInvisible(_) => "".into(), - token::Pound => "#".into(), - token::Dollar => "$".into(), - token::Question => "?".into(), - token::SingleQuote => "'".into(), + tk::At => "@".into(), + tk::Dot => ".".into(), + tk::DotDot => "..".into(), + tk::DotDotDot => "...".into(), + tk::DotDotEq => "..=".into(), + tk::Comma => ",".into(), + tk::Semi => ";".into(), + tk::Colon => ":".into(), + tk::PathSep => "::".into(), + tk::RArrow => "->".into(), + tk::LArrow => "<-".into(), + tk::FatArrow => "=>".into(), + tk::OpenParen => "(".into(), + tk::CloseParen => ")".into(), + tk::OpenBracket => "[".into(), + tk::CloseBracket => "]".into(), + tk::OpenBrace => "{".into(), + tk::CloseBrace => "}".into(), + tk::OpenInvisible(_) | tk::CloseInvisible(_) => "".into(), + tk::Pound => "#".into(), + tk::Dollar => "$".into(), + tk::Question => "?".into(), + tk::SingleQuote => "'".into(), /* Literals */ - token::Literal(lit) => literal_to_string(lit).into(), + tk::Literal(lit) => literal_to_string(lit).into(), /* Name components */ - token::Ident(name, is_raw) => { + tk::Ident(name, is_raw) => { IdentPrinter::new(name, is_raw.to_print_mode_ident(), convert_dollar_crate) .to_string() .into() } - token::NtIdent(ident, is_raw) => { + tk::NtIdent(ident, is_raw) => { IdentPrinter::for_ast_ident(ident, is_raw.to_print_mode_ident()).to_string().into() } - token::Lifetime(name, is_raw) | token::NtLifetime(Ident { name, .. }, is_raw) => { + tk::Lifetime(name, is_raw) | tk::NtLifetime(Ident { name, .. }, is_raw) => { IdentPrinter::new(name, is_raw.to_print_mode_lifetime(), None).to_string().into() } /* Other */ - token::DocComment(comment_kind, attr_style, data) => { - doc_comment_to_string(DocFragmentKind::Sugared(comment_kind), attr_style, data) + tk::DocComment(comment_kind, attr_style, data) => { + doc_comment_to_string(tk::DocFragmentKind::Sugared(comment_kind), attr_style, data) .into() } - token::Eof => "".into(), + tk::Eof => "".into(), } } /// Print the token precisely, without converting `$crate` into its respective crate name. - fn token_to_string(&self, token: &Token) -> Cow<'static, str> { + fn token_to_string(&self, token: &tk::Token) -> Cow<'static, str> { self.token_to_string_ext(token, false) } - fn token_to_string_ext(&self, token: &Token, convert_dollar_crate: bool) -> Cow<'static, str> { + fn token_to_string_ext( + &self, + token: &tk::Token, + convert_dollar_crate: bool, + ) -> Cow<'static, str> { let convert_dollar_crate = convert_dollar_crate.then_some(token.span); self.token_kind_to_string_ext(&token.kind, convert_dollar_crate) } @@ -2333,7 +2338,7 @@ impl<'a> State<'a> { self.print_token_literal(lit.as_token_lit(), lit.span) } - fn print_token_literal(&mut self, token_lit: token::Lit, span: Span) { + fn print_token_literal(&mut self, token_lit: tk::Lit, span: Span) { self.maybe_print_comment(span.lo()); self.word(token_lit.to_string()) } diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs index 5367de7a1cc82..7b52a5600ae2a 100644 --- a/compiler/rustc_expand/src/proc_macro_server.rs +++ b/compiler/rustc_expand/src/proc_macro_server.rs @@ -1,8 +1,7 @@ use std::ops::{Bound, Range}; -use ast::token::IdentIsRaw; use rustc_ast as ast; -use rustc_ast::token; +use rustc_ast::token as tk; use rustc_ast::tokenstream::{self, DelimSpacing, Spacing, TokenStream}; use rustc_ast::util::literal::escape_byte_str_symbol; use rustc_ast_pretty::pprust; @@ -31,65 +30,65 @@ trait ToInternal { fn to_internal(self) -> T; } -impl FromInternal for Delimiter { - fn from_internal(delim: token::Delimiter) -> Delimiter { +impl FromInternal for Delimiter { + fn from_internal(delim: tk::Delimiter) -> Delimiter { match delim { - token::Delimiter::Parenthesis => Delimiter::Parenthesis, - token::Delimiter::Brace => Delimiter::Brace, - token::Delimiter::Bracket => Delimiter::Bracket, - token::Delimiter::Invisible(_) => Delimiter::None, + tk::Delimiter::Parenthesis => Delimiter::Parenthesis, + tk::Delimiter::Brace => Delimiter::Brace, + tk::Delimiter::Bracket => Delimiter::Bracket, + tk::Delimiter::Invisible(_) => Delimiter::None, } } } -impl ToInternal for Delimiter { - fn to_internal(self) -> token::Delimiter { +impl ToInternal for Delimiter { + fn to_internal(self) -> tk::Delimiter { match self { - Delimiter::Parenthesis => token::Delimiter::Parenthesis, - Delimiter::Brace => token::Delimiter::Brace, - Delimiter::Bracket => token::Delimiter::Bracket, - Delimiter::None => token::Delimiter::Invisible(token::InvisibleOrigin::ProcMacro), + Delimiter::Parenthesis => tk::Delimiter::Parenthesis, + Delimiter::Brace => tk::Delimiter::Brace, + Delimiter::Bracket => tk::Delimiter::Bracket, + Delimiter::None => tk::Delimiter::Invisible(tk::InvisibleOrigin::ProcMacro), } } } -impl FromInternal for LitKind { - fn from_internal(kind: token::LitKind) -> Self { +impl FromInternal for LitKind { + fn from_internal(kind: tk::LitKind) -> Self { match kind { - token::Byte => LitKind::Byte, - token::Char => LitKind::Char, - token::Integer => LitKind::Integer, - token::Float => LitKind::Float, - token::Str => LitKind::Str, - token::StrRaw(n) => LitKind::StrRaw(n), - token::ByteStr => LitKind::ByteStr, - token::ByteStrRaw(n) => LitKind::ByteStrRaw(n), - token::CStr => LitKind::CStr, - token::CStrRaw(n) => LitKind::CStrRaw(n), - token::Err(_guar) => { + tk::Byte => LitKind::Byte, + tk::Char => LitKind::Char, + tk::Integer => LitKind::Integer, + tk::Float => LitKind::Float, + tk::Str => LitKind::Str, + tk::StrRaw(n) => LitKind::StrRaw(n), + tk::ByteStr => LitKind::ByteStr, + tk::ByteStrRaw(n) => LitKind::ByteStrRaw(n), + tk::CStr => LitKind::CStr, + tk::CStrRaw(n) => LitKind::CStrRaw(n), + tk::Err(_guar) => { // This is the only place a `rustc_proc_macro::bridge::LitKind::ErrWithGuar` // is constructed. Note that an `ErrorGuaranteed` is available, // as required. See the comment in `to_internal`. LitKind::ErrWithGuar } - token::Bool => unreachable!(), + tk::Bool => unreachable!(), } } } -impl ToInternal for LitKind { - fn to_internal(self) -> token::LitKind { +impl ToInternal for LitKind { + fn to_internal(self) -> tk::LitKind { match self { - LitKind::Byte => token::Byte, - LitKind::Char => token::Char, - LitKind::Integer => token::Integer, - LitKind::Float => token::Float, - LitKind::Str => token::Str, - LitKind::StrRaw(n) => token::StrRaw(n), - LitKind::ByteStr => token::ByteStr, - LitKind::ByteStrRaw(n) => token::ByteStrRaw(n), - LitKind::CStr => token::CStr, - LitKind::CStrRaw(n) => token::CStrRaw(n), + LitKind::Byte => tk::Byte, + LitKind::Char => tk::Char, + LitKind::Integer => tk::Integer, + LitKind::Float => tk::Float, + LitKind::Str => tk::Str, + LitKind::StrRaw(n) => tk::StrRaw(n), + LitKind::ByteStr => tk::ByteStr, + LitKind::ByteStrRaw(n) => tk::ByteStrRaw(n), + LitKind::CStr => tk::CStr, + LitKind::CStrRaw(n) => tk::CStrRaw(n), LitKind::ErrWithGuar => { // This is annoying but valid. `LitKind::ErrWithGuar` would // have an `ErrorGuaranteed` except that type isn't available @@ -98,7 +97,7 @@ impl ToInternal for LitKind { // which would be expensive. #[allow(deprecated)] let guar = ErrorGuaranteed::unchecked_error_guaranteed(); - token::Err(guar) + tk::Err(guar) } } } @@ -106,24 +105,22 @@ impl ToInternal for LitKind { impl FromInternal for Vec> { fn from_internal(stream: TokenStream) -> Self { - use rustc_ast::token::*; - // Estimate the capacity as `stream.len()` rounded up to the next power // of two to limit the number of required reallocations. let mut trees = Vec::with_capacity(stream.len().next_power_of_two()); for tree in stream.iter() { - let (Token { kind, span }, joint) = match tree.clone() { + let (tk::Token { kind, span }, joint) = match tree.clone() { tokenstream::TokenTree::Delimited(span, _, mut delim, mut stream) => { // In `mk_delimited` we avoid nesting invisible delimited // of the same `MetaVarKind`. Here we do the same but // ignore the `MetaVarKind` because it is discarded when we // convert it to a `Group`. - while let Delimiter::Invisible(InvisibleOrigin::MetaVar(_)) = delim + while let tk::Delimiter::Invisible(tk::InvisibleOrigin::MetaVar(_)) = delim && stream.len() == 1 && let tree = stream.get(0).unwrap() && let tokenstream::TokenTree::Delimited(_, _, delim2, stream2) = tree - && let Delimiter::Invisible(InvisibleOrigin::MetaVar(_)) = delim2 + && let tk::Delimiter::Invisible(tk::InvisibleOrigin::MetaVar(_)) = delim2 { delim = *delim2; stream = stream2.clone(); @@ -183,79 +180,79 @@ impl FromInternal for Vec> { }; match kind { - Eq => op("="), - Lt => op("<"), - Le => op("<="), - EqEq => op("=="), - Ne => op("!="), - Ge => op(">="), - Gt => op(">"), - AndAnd => op("&&"), - OrOr => op("||"), - Bang => op("!"), - Tilde => op("~"), - Plus => op("+"), - Minus => op("-"), - Star => op("*"), - Slash => op("/"), - Percent => op("%"), - Caret => op("^"), - And => op("&"), - Or => op("|"), - Shl => op("<<"), - Shr => op(">>"), - PlusEq => op("+="), - MinusEq => op("-="), - StarEq => op("*="), - SlashEq => op("/="), - PercentEq => op("%="), - CaretEq => op("^="), - AndEq => op("&="), - OrEq => op("|="), - ShlEq => op("<<="), - ShrEq => op(">>="), - At => op("@"), - Dot => op("."), - DotDot => op(".."), - DotDotDot => op("..."), - DotDotEq => op("..="), - Comma => op(","), - Semi => op(";"), - Colon => op(":"), - PathSep => op("::"), - RArrow => op("->"), - LArrow => op("<-"), - FatArrow => op("=>"), - Pound => op("#"), - Dollar => op("$"), - Question => op("?"), - SingleQuote => op("'"), - - Ident(sym, is_raw) => trees.push(TokenTree::Ident(Ident { + tk::Eq => op("="), + tk::Lt => op("<"), + tk::Le => op("<="), + tk::EqEq => op("=="), + tk::Ne => op("!="), + tk::Ge => op(">="), + tk::Gt => op(">"), + tk::AndAnd => op("&&"), + tk::OrOr => op("||"), + tk::Bang => op("!"), + tk::Tilde => op("~"), + tk::Plus => op("+"), + tk::Minus => op("-"), + tk::Star => op("*"), + tk::Slash => op("/"), + tk::Percent => op("%"), + tk::Caret => op("^"), + tk::And => op("&"), + tk::Or => op("|"), + tk::Shl => op("<<"), + tk::Shr => op(">>"), + tk::PlusEq => op("+="), + tk::MinusEq => op("-="), + tk::StarEq => op("*="), + tk::SlashEq => op("/="), + tk::PercentEq => op("%="), + tk::CaretEq => op("^="), + tk::AndEq => op("&="), + tk::OrEq => op("|="), + tk::ShlEq => op("<<="), + tk::ShrEq => op(">>="), + tk::At => op("@"), + tk::Dot => op("."), + tk::DotDot => op(".."), + tk::DotDotDot => op("..."), + tk::DotDotEq => op("..="), + tk::Comma => op(","), + tk::Semi => op(";"), + tk::Colon => op(":"), + tk::PathSep => op("::"), + tk::RArrow => op("->"), + tk::LArrow => op("<-"), + tk::FatArrow => op("=>"), + tk::Pound => op("#"), + tk::Dollar => op("$"), + tk::Question => op("?"), + tk::SingleQuote => op("'"), + + tk::Ident(sym, is_raw) => trees.push(TokenTree::Ident(Ident { sym, - is_raw: matches!(is_raw, IdentIsRaw::Yes), + is_raw: matches!(is_raw, tk::IdentIsRaw::Yes), span, })), - NtIdent(ident, is_raw) => trees.push(TokenTree::Ident(Ident { + tk::NtIdent(ident, is_raw) => trees.push(TokenTree::Ident(Ident { sym: ident.name, - is_raw: matches!(is_raw, IdentIsRaw::Yes), + is_raw: matches!(is_raw, tk::IdentIsRaw::Yes), span: ident.span, })), - Lifetime(name, is_raw) => { + tk::Lifetime(name, is_raw) => { let ident = rustc_span::Ident::new(name, span).without_first_quote(); trees.extend([ TokenTree::Punct(Punct { ch: b'\'', joint: true, span }), TokenTree::Ident(Ident { sym: ident.name, - is_raw: matches!(is_raw, IdentIsRaw::Yes), + is_raw: matches!(is_raw, tk::IdentIsRaw::Yes), span, }), ]); } - NtLifetime(ident, is_raw) => { + tk::NtLifetime(ident, is_raw) => { let stream = - TokenStream::token_alone(token::Lifetime(ident.name, is_raw), ident.span); + TokenStream::token_alone(tk::Lifetime(ident.name, is_raw), ident.span); trees.push(TokenTree::Group(Group { delimiter: rustc_proc_macro::Delimiter::None, stream: Some(stream), @@ -263,7 +260,7 @@ impl FromInternal for Vec> { })) } - Literal(token::Lit { kind, symbol, suffix }) => { + tk::Literal(tk::Lit { kind, symbol, suffix }) => { trees.push(TokenTree::Literal(self::Literal { kind: FromInternal::from_internal(kind), symbol, @@ -271,15 +268,15 @@ impl FromInternal for Vec> { span, })); } - DocComment(_, attr_style, data) => { + tk::DocComment(_, attr_style, data) => { let mut escaped = String::new(); for ch in data.as_str().chars() { escaped.extend(ch.escape_debug()); } let stream = [ - Ident(sym::doc, IdentIsRaw::No), - Eq, - TokenKind::lit(token::Str, Symbol::intern(&escaped), None), + tk::Ident(sym::doc, tk::IdentIsRaw::No), + tk::Eq, + tk::TokenKind::lit(tk::Str, Symbol::intern(&escaped), None), ] .into_iter() .map(|kind| tokenstream::TokenTree::token_alone(kind, span)) @@ -295,8 +292,15 @@ impl FromInternal for Vec> { })); } - OpenParen | CloseParen | OpenBrace | CloseBrace | OpenBracket | CloseBracket - | OpenInvisible(_) | CloseInvisible(_) | Eof => unreachable!(), + tk::OpenParen + | tk::CloseParen + | tk::OpenBrace + | tk::CloseBrace + | tk::OpenBracket + | tk::CloseBracket + | tk::OpenInvisible(_) + | tk::CloseInvisible(_) + | tk::Eof => unreachable!(), } } trees @@ -308,8 +312,6 @@ impl ToInternal> for (TokenTree, &mut Rustc<'_, '_>) { fn to_internal(self) -> SmallVec<[tokenstream::TokenTree; 2]> { - use rustc_ast::token::*; - // The code below is conservative, using `token_alone`/`Spacing::Alone` // in most places. It's hard in general to do better when working at // the token level. When the resulting code is pretty-printed by @@ -319,31 +321,31 @@ impl ToInternal> match tree { TokenTree::Punct(Punct { ch, joint, span }) => { let kind = match ch { - b'=' => Eq, - b'<' => Lt, - b'>' => Gt, - b'!' => Bang, - b'~' => Tilde, - b'+' => Plus, - b'-' => Minus, - b'*' => Star, - b'/' => Slash, - b'%' => Percent, - b'^' => Caret, - b'&' => And, - b'|' => Or, - b'@' => At, - b'.' => Dot, - b',' => Comma, - b';' => Semi, - b':' => Colon, - b'#' => Pound, - b'$' => Dollar, - b'?' => Question, - b'\'' => SingleQuote, + b'=' => tk::Eq, + b'<' => tk::Lt, + b'>' => tk::Gt, + b'!' => tk::Bang, + b'~' => tk::Tilde, + b'+' => tk::Plus, + b'-' => tk::Minus, + b'*' => tk::Star, + b'/' => tk::Slash, + b'%' => tk::Percent, + b'^' => tk::Caret, + b'&' => tk::And, + b'|' => tk::Or, + b'@' => tk::At, + b'.' => tk::Dot, + b',' => tk::Comma, + b';' => tk::Semi, + b':' => tk::Colon, + b'#' => tk::Pound, + b'$' => tk::Dollar, + b'?' => tk::Question, + b'\'' => tk::SingleQuote, _ => unreachable!(), }; - // We never produce `token::Spacing::JointHidden` here, which + // We never produce `tk::Spacing::JointHidden` here, which // means the pretty-printing of code produced by proc macros is // ugly, with lots of whitespace between tokens. This is // unavoidable because `proc_macro::Spacing` only applies to @@ -364,7 +366,7 @@ impl ToInternal> } TokenTree::Ident(self::Ident { sym, is_raw, span }) => { rustc.psess().symbol_gallery.insert(sym, span); - smallvec![tokenstream::TokenTree::token_alone(Ident(sym, is_raw.into()), span)] + smallvec![tokenstream::TokenTree::token_alone(tk::Ident(sym, is_raw.into()), span)] } TokenTree::Literal(self::Literal { kind: self::LitKind::Integer, @@ -373,8 +375,8 @@ impl ToInternal> span, }) if let Some(symbol) = symbol.as_str().strip_prefix('-') => { let symbol = Symbol::intern(symbol); - let integer = TokenKind::lit(token::Integer, symbol, suffix); - let a = tokenstream::TokenTree::token_joint_hidden(Minus, span); + let integer = tk::TokenKind::lit(tk::Integer, symbol, suffix); + let a = tokenstream::TokenTree::token_joint_hidden(tk::Minus, span); let b = tokenstream::TokenTree::token_alone(integer, span); smallvec![a, b] } @@ -385,14 +387,14 @@ impl ToInternal> span, }) if let Some(symbol) = symbol.as_str().strip_prefix('-') => { let symbol = Symbol::intern(symbol); - let float = TokenKind::lit(token::Float, symbol, suffix); - let a = tokenstream::TokenTree::token_joint_hidden(Minus, span); + let float = tk::TokenKind::lit(tk::Float, symbol, suffix); + let a = tokenstream::TokenTree::token_joint_hidden(tk::Minus, span); let b = tokenstream::TokenTree::token_alone(float, span); smallvec![a, b] } TokenTree::Literal(self::Literal { kind, symbol, suffix, span }) => { smallvec![tokenstream::TokenTree::token_alone( - TokenKind::lit(kind.to_internal(), symbol, suffix), + tk::TokenKind::lit(kind.to_internal(), symbol, suffix), span, )] } @@ -500,7 +502,7 @@ impl server::Server for Rustc<'_, '_> { let minus_present = parser.eat(exp!(Minus)); let lit_span = parser.token.span.data(); - let token::Literal(mut lit) = parser.token.kind else { + let tk::Literal(mut lit) = parser.token.kind else { return Err("not a literal".to_string()); }; @@ -519,26 +521,26 @@ impl server::Server for Rustc<'_, '_> { // Check literal is a kind we allow to be negated in a proc macro token. match lit.kind { - token::LitKind::Bool - | token::LitKind::Byte - | token::LitKind::Char - | token::LitKind::Str - | token::LitKind::StrRaw(_) - | token::LitKind::ByteStr - | token::LitKind::ByteStrRaw(_) - | token::LitKind::CStr - | token::LitKind::CStrRaw(_) - | token::LitKind::Err(_) => { + tk::LitKind::Bool + | tk::LitKind::Byte + | tk::LitKind::Char + | tk::LitKind::Str + | tk::LitKind::StrRaw(_) + | tk::LitKind::ByteStr + | tk::LitKind::ByteStrRaw(_) + | tk::LitKind::CStr + | tk::LitKind::CStrRaw(_) + | tk::LitKind::Err(_) => { return Err("non-numeric literal may not be negated".to_string()); } - token::LitKind::Integer | token::LitKind::Float => {} + tk::LitKind::Integer | tk::LitKind::Float => {} } // Synthesize a new symbol that includes the minus sign. let symbol = Symbol::intern(&s[..1 + lit.symbol.as_str().len()]); - lit = token::Lit::new(lit.kind, symbol, lit.suffix); + lit = tk::Lit::new(lit.kind, symbol, lit.suffix); } - let token::Lit { kind, symbol, suffix } = lit; + let tk::Lit { kind, symbol, suffix } = lit; Ok(Literal { kind: FromInternal::from_internal(kind), symbol, @@ -592,7 +594,7 @@ impl server::Server for Rustc<'_, '_> { let expr = try { let mut p = Parser::new(self.psess(), stream.clone(), Some("proc_macro expand expr")); let expr = p.parse_expr()?; - if p.token != token::Eof { + if p.token != tk::Eof { p.unexpected()?; } expr @@ -613,31 +615,28 @@ impl server::Server for Rustc<'_, '_> { // We don't use `TokenStream::from_ast` as the tokenstream currently cannot // be recovered in the general case. match &expr.kind { - ast::ExprKind::Lit(token_lit) if token_lit.kind == token::Bool => { + ast::ExprKind::Lit(token_lit) if token_lit.kind == tk::Bool => { Ok(tokenstream::TokenStream::token_alone( - token::Ident(token_lit.symbol, IdentIsRaw::No), + tk::Ident(token_lit.symbol, tk::IdentIsRaw::No), expr.span, )) } ast::ExprKind::Lit(token_lit) => { - Ok(tokenstream::TokenStream::token_alone(token::Literal(*token_lit), expr.span)) + Ok(tokenstream::TokenStream::token_alone(tk::Literal(*token_lit), expr.span)) } ast::ExprKind::IncludedBytes(byte_sym) => { - let lit = token::Lit::new( - token::ByteStr, - escape_byte_str_symbol(byte_sym.as_byte_str()), - None, - ); - Ok(tokenstream::TokenStream::token_alone(token::TokenKind::Literal(lit), expr.span)) + let lit = + tk::Lit::new(tk::ByteStr, escape_byte_str_symbol(byte_sym.as_byte_str()), None); + Ok(tokenstream::TokenStream::token_alone(tk::TokenKind::Literal(lit), expr.span)) } ast::ExprKind::Unary(ast::UnOp::Neg, e) => match &e.kind { ast::ExprKind::Lit(token_lit) => match token_lit { - token::Lit { kind: token::Integer | token::Float, .. } => { + tk::Lit { kind: tk::Integer | tk::Float, .. } => { Ok(Self::TokenStream::from_iter([ // FIXME: The span of the `-` token is lost when // parsing, so we cannot faithfully recover it here. - tokenstream::TokenTree::token_joint_hidden(token::Minus, e.span), - tokenstream::TokenTree::token_alone(token::Literal(*token_lit), e.span), + tokenstream::TokenTree::token_joint_hidden(tk::Minus, e.span), + tokenstream::TokenTree::token_alone(tk::Literal(*token_lit), e.span), ])) } _ => Err(()),