From 3f2624ea64d1077e10b0f0a12f2f5253c019ebfe Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Mon, 20 Jul 2026 08:52:20 +1000 Subject: [PATCH 1/2] Make `TokenTreeCursor` private And also the fields of `TokenCursor`. It's good hygiene in general, and will allow larger changes to `TokenCursor` down the road (e.g. hopefully avoiding the flattening of token trees to a linear token stream). --- compiler/rustc_ast/src/tokenstream.rs | 66 ++++++++++++++++++++++--- compiler/rustc_parse/src/parser/expr.rs | 9 ++-- compiler/rustc_parse/src/parser/mod.rs | 31 +++++------- 3 files changed, 74 insertions(+), 32 deletions(-) diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs index 19ac92aa66fe7..61c899188f82a 100644 --- a/compiler/rustc_ast/src/tokenstream.rs +++ b/compiler/rustc_ast/src/tokenstream.rs @@ -880,7 +880,7 @@ impl<'t> Iterator for TokenStreamIter<'t> { } #[derive(Clone, Debug)] -pub struct TokenTreeCursor { +struct TokenTreeCursor { stream: TokenStream, /// Points to the current token tree in the stream. In `TokenCursor::curr`, /// this can be any token tree. In `TokenCursor::stack`, this is always a @@ -890,27 +890,27 @@ pub struct TokenTreeCursor { impl TokenTreeCursor { #[inline] - pub fn new(stream: TokenStream) -> Self { + fn new(stream: TokenStream) -> Self { TokenTreeCursor { stream, index: 0 } } #[inline] - pub fn curr(&self) -> Option<&TokenTree> { + fn curr(&self) -> Option<&TokenTree> { self.stream.get(self.index) } - pub fn look_ahead(&self, n: usize) -> Option<&TokenTree> { + fn look_ahead(&self, n: usize) -> Option<&TokenTree> { self.stream.get(self.index + n) } #[inline] - pub fn bump(&mut self) { + fn bump(&mut self) { self.index += 1; } // For skipping ahead in rare circumstances. #[inline] - pub fn bump_to_end(&mut self) { + fn bump_to_end(&mut self) { self.index = self.stream.len(); } } @@ -926,18 +926,68 @@ pub struct TokenCursor { // The delimiters for this token stream are found in `self.stack.last()`; // if that is `None` we are in the outermost token stream which never has // delimiters. - pub curr: TokenTreeCursor, + curr: TokenTreeCursor, // Token streams surrounding the current one. The index within each cursor // always points to a `TokenTree::Delimited`. - pub stack: Vec, + stack: Vec, } impl TokenCursor { + #[inline] + pub fn new(stream: TokenStream) -> Self { + TokenCursor { curr: TokenTreeCursor::new(stream), stack: vec![] } + } + pub fn next(&mut self) -> (Token, Spacing) { self.inlined_next() } + /// An `n` of zero is the next token tree in the current token stream; won't look outside the + /// current token stream. + #[inline] + pub fn look_ahead(&self, n: usize) -> Option<&TokenTree> { + self.curr.look_ahead(n) + } + + /// Returns the first token tree (if there is one) past the close delimiter of the enclosing + /// delimited sequence. Panics if we are not within a delimited sequence. + #[inline] + pub fn look_ahead_past_close_delim(&self) -> Option<&TokenTree> { + self.stack.last().unwrap().look_ahead(1) + } + + /// Clones the `TokenTree::Delimited` that we are currently within. Panics if we are not within + /// a delimited sequence. + #[inline] + pub fn clone_enclosing_delim(&self) -> TokenTree { + self.stack.last().unwrap().curr().unwrap().clone() + } + + /// For skipping to the end of the current sequence, in rare circumstances. + #[inline] + pub fn bump_to_end(&mut self) { + self.curr.bump_to_end() + } + + /// Note: the outermost stream has depth of 0. + #[inline] + pub fn depth(&self) -> usize { + self.stack.len() + } + + /// Returns details about the parent delimited sequence, if there is one. + #[inline] + pub fn parent_delim_and_span(&self) -> Option<(Delimiter, DelimSpan)> { + if let Some(last) = self.stack.last() + && let Some(TokenTree::Delimited(span, _, delim, _)) = last.curr() + { + Some((*delim, *span)) + } else { + None + } + } + /// This always-inlined version should only be used on hot code paths. #[inline(always)] pub fn inlined_next(&mut self) -> (Token, Spacing) { diff --git a/compiler/rustc_parse/src/parser/expr.rs b/compiler/rustc_parse/src/parser/expr.rs index 92e8f5eefadb2..c5757f36b2e6a 100644 --- a/compiler/rustc_parse/src/parser/expr.rs +++ b/compiler/rustc_parse/src/parser/expr.rs @@ -7,7 +7,6 @@ use ast::mut_visit::{self, MutVisitor}; use ast::token::IdentIsRaw; use ast::{CoroutineKind, ForLoopKind, GenBlockKind, MatchKind, Pat, Path, PathSegment, Recovered}; use rustc_ast::token::{self, Delimiter, InvisibleOrigin, MetaVarKind, Token, TokenKind}; -use rustc_ast::tokenstream::TokenTree; use rustc_ast::util::case::Case; use rustc_ast::util::classify; use rustc_ast::util::parser::{AssocOp, ExprPrecedence, Fixity, prec_let_scrutinee_needs_par}; @@ -1276,13 +1275,12 @@ impl<'a> Parser<'a> { None }; let open_paren = self.token.span; - let call_depth = self.token_cursor.stack.len(); + let call_depth = self.token_cursor.depth(); let seq = match self.parse_expr_paren_seq() { Ok(args) => Ok(self.mk_expr(lo.to(self.prev_token.span), self.mk_call(fun, args))), Err(err) - if self.is_expected_raw_ref_mut() - && self.token_cursor.stack.len() == call_depth => + if self.is_expected_raw_ref_mut() && self.token_cursor.depth() == call_depth => { let guar = err.emit(); // Preserve the call expression so later passes can still diagnose the callee, @@ -2510,8 +2508,7 @@ impl<'a> Parser<'a> { } if self.token == TokenKind::Semi - && let Some(last) = self.token_cursor.stack.last() - && let Some(TokenTree::Delimited(_, _, Delimiter::Parenthesis, _)) = last.curr() + && let Some((Delimiter::Parenthesis, _)) = self.token_cursor.parent_delim_and_span() && self.may_recover() { // It is likely that the closure body is a block but where the diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs index 09ac1acb74f51..5ef0fb04cd81f 100644 --- a/compiler/rustc_parse/src/parser/mod.rs +++ b/compiler/rustc_parse/src/parser/mod.rs @@ -29,8 +29,7 @@ use rustc_ast::token::{ self, IdentIsRaw, InvisibleOrigin, MetaVarKind, NtExprKind, NtPatKind, Token, TokenKind, }; use rustc_ast::tokenstream::{ - ParserRange, ParserReplacement, Spacing, TokenCursor, TokenStream, TokenTree, TokenTreeCursor, - WithTokens, + ParserRange, ParserReplacement, Spacing, TokenCursor, TokenStream, TokenTree, WithTokens, }; use rustc_ast::util::case::Case; use rustc_ast::util::classify; @@ -346,7 +345,7 @@ impl<'a> Parser<'a> { ) -> Self { let mut parser = Parser { psess, - token_cursor: TokenCursor { curr: TokenTreeCursor::new(stream), stack: Vec::new() }, + token_cursor: TokenCursor::new(stream), subparser_name, capture_state: CaptureState { capturing: Capturing::No, @@ -502,10 +501,8 @@ impl<'a> Parser<'a> { // Primarily used when `self.token` matches `OpenInvisible(_))`, to look // ahead through the current metavar expansion. fn check_noexpect_past_close_delim(&self, tok: &TokenKind) -> bool { - let mut tree_cursor = self.token_cursor.stack.last().unwrap().clone(); - tree_cursor.bump(); matches!( - tree_cursor.curr(), + self.token_cursor.look_ahead_past_close_delim(), Some(TokenTree::Token(token::Token { kind, .. }, _)) if kind == tok ) } @@ -1157,9 +1154,8 @@ impl<'a> Parser<'a> { // Typically around 98% of the `dist > 0` cases have `dist == 1`, so we // have a fast special case for that. if dist == 1 { - // The index is zero because the tree cursor's index always points - // to the next token to be gotten. - match self.token_cursor.curr.curr() { + // `look_ahead(0)` returns the *next* token. + match self.token_cursor.look_ahead(0) { Some(tree) => { // Indexing stayed within the current token tree. match tree { @@ -1174,8 +1170,7 @@ impl<'a> Parser<'a> { None => { // The tree cursor lookahead went (one) past the end of the // current token tree. Try to return a close delimiter. - if let Some(last) = self.token_cursor.stack.last() - && let Some(&TokenTree::Delimited(span, _, delim, _)) = last.curr() + if let Some((delim, span)) = self.token_cursor.parent_delim_and_span() && !delim.skip() { // We are not in the outermost token stream, so we have @@ -1203,7 +1198,7 @@ impl<'a> Parser<'a> { looker(&token) } - /// Like `lookahead`, but skips over token trees rather than tokens. Useful + /// Like `look_ahead`, but skips over token trees rather than tokens. Useful /// when looking past possible metavariable pasting sites. pub fn tree_look_ahead( &self, @@ -1211,7 +1206,7 @@ impl<'a> Parser<'a> { looker: impl FnOnce(&TokenTree) -> R, ) -> Option { assert_ne!(dist, 0); - self.token_cursor.curr.look_ahead(dist - 1).map(looker) + self.token_cursor.look_ahead(dist - 1).map(looker) } /// Returns whether any of the given keywords are `dist` tokens ahead of the current one. @@ -1408,7 +1403,7 @@ impl<'a> Parser<'a> { if self.token.kind.open_delim().is_some() { // Clone the `TokenTree::Delimited` that we are currently // within. That's what we are going to return. - let tree = self.token_cursor.stack.last().unwrap().curr().unwrap().clone(); + let tree = self.token_cursor.clone_enclosing_delim(); debug_assert_matches!(tree, TokenTree::Delimited(..)); // Advance the token cursor through the entire delimited @@ -1416,22 +1411,22 @@ impl<'a> Parser<'a> { // delimited sequence, i.e. at depth `d`. After getting the // matching `CloseDelim` we are *after* the delimited sequence, // i.e. at depth `d - 1`. - let target_depth = self.token_cursor.stack.len() - 1; + let target_depth = self.token_cursor.depth() - 1; if let Capturing::No = self.capture_state.capturing { // We are not capturing tokens, so skip to the end of the // delimited sequence. This is a perf win when dealing with // declarative macros that pass large `tt` fragments through // multiple rules, as seen in the uom-0.37.0 crate. - self.token_cursor.curr.bump_to_end(); + self.token_cursor.bump_to_end(); self.bump(); - debug_assert_eq!(self.token_cursor.stack.len(), target_depth); + debug_assert_eq!(self.token_cursor.depth(), target_depth); } else { loop { // Advance one token at a time, so `TokenCursor::next()` // can capture these tokens if necessary. self.bump(); - if self.token_cursor.stack.len() == target_depth { + if self.token_cursor.depth() == target_depth { break; } } From 73de522410e104c9aef116e79abdf80fdc1c88c5 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Mon, 20 Jul 2026 21:44:41 +1000 Subject: [PATCH 2/2] Make `TokenStream`'s internals private --- compiler/rustc_ast/src/tokenstream.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs index 61c899188f82a..b6fb6f33e3984 100644 --- a/compiler/rustc_ast/src/tokenstream.rs +++ b/compiler/rustc_ast/src/tokenstream.rs @@ -622,7 +622,7 @@ pub enum Spacing { /// A `TokenStream` is an abstract sequence of tokens, organized into [`TokenTree`]s. #[derive(Clone, Debug, Default, PartialEq, Eq, Hash, Encodable, Decodable)] -pub struct TokenStream(pub(crate) Arc>); +pub struct TokenStream(Arc>); impl TokenStream { pub fn new(tts: Vec) -> TokenStream {