From 47e62ee30c6e6c4dd13296dd671c9d1126515324 Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Thu, 9 Jul 2026 09:32:55 +1000
Subject: [PATCH 1/2] Add a unit test for `TokenStreamIter`.

This shows that `size_hint` is imprecise.
---
 compiler/rustc_ast/src/tokenstream.rs       |  3 +++
 compiler/rustc_ast/src/tokenstream/tests.rs | 13 +++++++++++++
 2 files changed, 16 insertions(+)
 create mode 100644 compiler/rustc_ast/src/tokenstream/tests.rs

diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs
index baad20a86784d..1ef91d718a810 100644
--- a/compiler/rustc_ast/src/tokenstream.rs
+++ b/compiler/rustc_ast/src/tokenstream.rs
@@ -22,6 +22,9 @@ use crate::ast_traits::{HasAttrs, HasTokens};
 use crate::token::{self, Delimiter, Token, TokenKind};
 use crate::{AttrVec, Attribute};
 
+#[cfg(test)]
+mod tests;
+
 /// Part of a `TokenStream`.
 #[derive(Debug, Clone, PartialEq, Eq, Hash, Encodable, Decodable, StableHash)]
 pub enum TokenTree {
diff --git a/compiler/rustc_ast/src/tokenstream/tests.rs b/compiler/rustc_ast/src/tokenstream/tests.rs
new file mode 100644
index 0000000000000..9b79e5fabf1d9
--- /dev/null
+++ b/compiler/rustc_ast/src/tokenstream/tests.rs
@@ -0,0 +1,13 @@
+use rustc_span::DUMMY_SP;
+
+use crate::token::TokenKind;
+use crate::tokenstream::TokenStream;
+
+#[test]
+fn test_token_stream_iter() {
+    let ts = TokenStream::token_alone(TokenKind::Eq, DUMMY_SP);
+    assert_eq!(ts.len(), 1);
+
+    let iter = ts.iter();
+    assert_eq!(iter.size_hint(), (0, None));
+}

From fa43a690519ca7ea158cd23ba451f27dcef07e46 Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Thu, 9 Jul 2026 09:25:17 +1000
Subject: [PATCH 2/2] Improve `TokenStreamIter`.

It's simpler if it's just a wrapper around `slice::Iter`.

Also, we might as well implement `size_hint`. It doesn't have any effect
on perf but it's trivial and might help one day.
---
 compiler/rustc_ast/src/tokenstream.rs       | 18 ++++++++----------
 compiler/rustc_ast/src/tokenstream/tests.rs |  2 +-
 2 files changed, 9 insertions(+), 11 deletions(-)

diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs
index 1ef91d718a810..98caac2c88f77 100644
--- a/compiler/rustc_ast/src/tokenstream.rs
+++ b/compiler/rustc_ast/src/tokenstream.rs
@@ -836,21 +836,18 @@ impl StableHash for TokenStream {
 }
 
 #[derive(Clone)]
-pub struct TokenStreamIter<'t> {
-    stream: &'t TokenStream,
-    index: usize,
-}
+pub struct TokenStreamIter<'t>(std::slice::Iter<'t, TokenTree>);
 
 impl<'t> TokenStreamIter<'t> {
     fn new(stream: &'t TokenStream) -> Self {
-        TokenStreamIter { stream, index: 0 }
+        TokenStreamIter(stream.0.as_slice().iter())
     }
 
     // Peeking could be done via `Peekable`, but most iterators need peeking,
     // and this is simple and avoids the need to use `peekable` and `Peekable`
     // at all the use sites.
     pub fn peek(&self) -> Option<&'t TokenTree> {
-        self.stream.0.get(self.index)
+        self.0.as_slice().first()
     }
 }
 
@@ -858,10 +855,11 @@ impl<'t> Iterator for TokenStreamIter<'t> {
     type Item = &'t TokenTree;
 
     fn next(&mut self) -> Option<&'t TokenTree> {
-        self.stream.0.get(self.index).map(|tree| {
-            self.index += 1;
-            tree
-        })
+        self.0.next()
+    }
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        self.0.size_hint()
     }
 }
 
diff --git a/compiler/rustc_ast/src/tokenstream/tests.rs b/compiler/rustc_ast/src/tokenstream/tests.rs
index 9b79e5fabf1d9..6c7e82a97c58e 100644
--- a/compiler/rustc_ast/src/tokenstream/tests.rs
+++ b/compiler/rustc_ast/src/tokenstream/tests.rs
@@ -9,5 +9,5 @@ fn test_token_stream_iter() {
     assert_eq!(ts.len(), 1);
 
     let iter = ts.iter();
-    assert_eq!(iter.size_hint(), (0, None));
+    assert_eq!(iter.size_hint(), (1, Some(1)));
 }