WIP: Move jointness to Token

Author: matklad

compiler/rustc_ast/src/attr/mod.rs

@@ -8,7 +8,7 @@ use crate::ast::{Path, PathSegment};
 use crate::mut_visit::visit_clobber;
 use crate::ptr::P;
 use crate::token::{self, CommentKind, Token};
-use crate::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndJoint};
+use crate::tokenstream::{DelimSpan, TokenStream, TokenTree};
 
 use rustc_index::bit_set::GrowableBitSet;
 use rustc_span::source_map::{BytePos, Spanned};
@@ -362,7 +362,7 @@ pub fn list_contains_name(items: &[NestedMetaItem], name: Symbol) -> bool {
 }
 
 impl MetaItem {
-    fn token_trees_and_joints(&self) -> Vec<TreeAndJoint> {
+    fn token_trees_and_joints(&self) -> Vec<TokenTree> {
         let mut idents = vec![];
         let mut last_pos = BytePos(0 as u32);
         for (i, segment) in self.path.segments.iter().enumerate() {
@@ -388,6 +388,7 @@ impl MetaItem {
             Some(TokenTree::Token(Token {
                 kind: kind @ (token::Ident(..) | token::ModSep),
                 span,
+                ..
             })) => 'arm: {
                 let mut segments = if let token::Ident(name, _) = kind {
                     if let Some(TokenTree::Token(Token { kind: token::ModSep, .. })) = tokens.peek()
@@ -401,8 +402,9 @@ impl MetaItem {
                     vec![PathSegment::path_root(span)]
                 };
                 loop {
-                    if let Some(TokenTree::Token(Token { kind: token::Ident(name, _), span })) =
-                        tokens.next().map(TokenTree::uninterpolate)
+                    if let Some(TokenTree::Token(Token {
+                        kind: token::Ident(name, _), span, ..
+                    })) = tokens.next().map(TokenTree::uninterpolate)
                     {
                         segments.push(PathSegment::from_ident(Ident::new(name, span)));
                     } else {
@@ -459,7 +461,7 @@ impl MetaItemKind {
         }
     }
 
-    fn token_trees_and_joints(&self, span: Span) -> Vec<TreeAndJoint> {
+    fn token_trees_and_joints(&self, span: Span) -> Vec<TokenTree> {
         match *self {
             MetaItemKind::Word => vec![],
             MetaItemKind::NameValue(ref lit) => {
@@ -554,7 +556,7 @@ impl NestedMetaItem {
         }
     }
 
-    fn token_trees_and_joints(&self) -> Vec<TreeAndJoint> {
+    fn token_trees_and_joints(&self) -> Vec<TokenTree> {
         match *self {
             NestedMetaItem::MetaItem(ref item) => item.token_trees_and_joints(),
             NestedMetaItem::Literal(ref lit) => vec![lit.token_tree().into()],

compiler/rustc_ast/src/mut_visit.rs

@@ -641,14 +641,14 @@ pub fn noop_visit_tt<T: MutVisitor>(tt: &mut TokenTree, vis: &mut T) {
 
 pub fn noop_visit_tts<T: MutVisitor>(TokenStream(tts): &mut TokenStream, vis: &mut T) {
     let tts = Lrc::make_mut(tts);
-    visit_vec(tts, |(tree, _is_joint)| vis.visit_tt(tree));
+    visit_vec(tts, |tree| vis.visit_tt(tree));
 }
 
 // Applies ident visitor if it's an ident; applies other visits to interpolated nodes.
 // In practice the ident part is not actually used by specific visitors right now,
 // but there's a test below checking that it works.
 pub fn noop_visit_token<T: MutVisitor>(t: &mut Token, vis: &mut T) {
-    let Token { kind, span } = t;
+    let Token { kind, span, is_joint: _ } = t;
     match kind {
         token::Ident(name, _) | token::Lifetime(name) => {
             let mut ident = Ident::new(*name, *span);

compiler/rustc_ast/src/token.rs

@@ -6,7 +6,7 @@ pub use TokenKind::*;
 
 use crate::ast;
 use crate::ptr::P;
-use crate::tokenstream::TokenTree;
+use crate::tokenstream::{IsJoint, TokenTree};
 
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_data_structures::sync::Lrc;
@@ -262,6 +262,13 @@ rustc_data_structures::static_assert_size!(TokenKind, 16);
 pub struct Token {
     pub kind: TokenKind,
     pub span: Span,
+    /// Is this operator token immediately followed by another operator token,
+    /// without whitespace?
+    ///
+    /// At the moment, this field is only used for proc_macros, and joint tokens
+    /// are usually represented by dedicated token kinds. We want to transition
+    /// to using jointness everywhere though (#63689).
+    pub is_joint: IsJoint,
 }
 
 impl TokenKind {
@@ -315,7 +322,11 @@ impl TokenKind {
 
 impl Token {
     pub fn new(kind: TokenKind, span: Span) -> Self {
-        Token { kind, span }
+        Token::with_spacing(kind, span, IsJoint::NonJoint)
+    }
+
+    pub fn with_spacing(kind: TokenKind, span: Span, is_joint: IsJoint) -> Self {
+        Token { kind, span, is_joint }
     }
 
     /// Some token that will be thrown away later.
@@ -613,6 +624,10 @@ impl Token {
     }
 
     pub fn glue(&self, joint: &Token) -> Option<Token> {
+        if self.is_joint == IsJoint::NonJoint {
+            return None;
+        }
+
         let kind = match self.kind {
             Eq => match joint.kind {
                 Eq => EqEq,
@@ -668,7 +683,9 @@ impl Token {
             | Lifetime(..) | Interpolated(..) | DocComment(..) | Eof => return None,
         };
 
-        Some(Token::new(kind, self.span.to(joint.span)))
+        let mut token = Token::new(kind, self.span.to(joint.span));
+        token.is_joint = joint.is_joint;
+        Some(token)
     }
 }
 
@@ -699,7 +716,7 @@ pub enum Nonterminal {
 
 // `Nonterminal` is used a lot. Make sure it doesn't unintentionally get bigger.
 #[cfg(target_arch = "x86_64")]
-rustc_data_structures::static_assert_size!(Nonterminal, 40);
+rustc_data_structures::static_assert_size!(Nonterminal, 48);
 
 #[derive(Debug, Copy, Clone, PartialEq, Encodable, Decodable)]
 pub enum NonterminalKind {

compiler/rustc_ast/src/tokenstream.rs

@@ -82,10 +82,6 @@ impl TokenTree {
         }
     }
 
-    pub fn joint(self) -> TokenStream {
-        TokenStream::new(vec![(self, Joint)])
-    }
-
     pub fn token(kind: TokenKind, span: Span) -> TokenTree {
         TokenTree::Token(Token::new(kind, span))
     }
@@ -125,22 +121,18 @@ where
 /// instead of a representation of the abstract syntax tree.
 /// Today's `TokenTree`s can still contain AST via `token::Interpolated` for back-compat.
 #[derive(Clone, Debug, Default, Encodable, Decodable)]
-pub struct TokenStream(pub Lrc<Vec<TreeAndJoint>>);
-
-pub type TreeAndJoint = (TokenTree, IsJoint);
+pub struct TokenStream(pub Lrc<Vec<TokenTree>>);
 
 // `TokenStream` is used a lot. Make sure it doesn't unintentionally get bigger.
 #[cfg(target_arch = "x86_64")]
 rustc_data_structures::static_assert_size!(TokenStream, 8);
 
-#[derive(Clone, Copy, Debug, PartialEq, Encodable, Decodable)]
+#[derive(Clone, Copy, Debug, PartialEq, Encodable, Decodable, HashStable_Generic)]
 pub enum IsJoint {
     Joint,
     NonJoint,
 }
 
-use IsJoint::*;
-
 impl TokenStream {
     /// Given a `TokenStream` with a `Stream` of only two arguments, return a new `TokenStream`
     /// separating the two arguments with a comma for diagnostic suggestions.
@@ -151,22 +143,22 @@ impl TokenStream {
         while let Some((pos, ts)) = iter.next() {
             if let Some((_, next)) = iter.peek() {
                 let sp = match (&ts, &next) {
-                    (_, (TokenTree::Token(Token { kind: token::Comma, .. }), _)) => continue,
+                    (_, TokenTree::Token(Token { kind: token::Comma, .. })) => continue,
                     (
-                        (TokenTree::Token(token_left), NonJoint),
-                        (TokenTree::Token(token_right), _),
+                        TokenTree::Token(token_left @ Token { is_joint: IsJoint::NonJoint, .. }),
+                        TokenTree::Token(token_right),
                     ) if ((token_left.is_ident() && !token_left.is_reserved_ident())
                         || token_left.is_lit())
                         && ((token_right.is_ident() && !token_right.is_reserved_ident())
                             || token_right.is_lit()) =>
                     {
                         token_left.span
                     }
-                    ((TokenTree::Delimited(sp, ..), NonJoint), _) => sp.entire(),
+                    (TokenTree::Delimited(sp, ..), _) => sp.entire(),
                     _ => continue,
                 };
                 let sp = sp.shrink_to_hi();
-                let comma = (TokenTree::token(token::Comma, sp), NonJoint);
+                let comma = TokenTree::token(token::Comma, sp);
                 suggestion = Some((pos, comma, sp));
             }
         }
@@ -184,19 +176,13 @@ impl TokenStream {
 
 impl From<TokenTree> for TokenStream {
     fn from(tree: TokenTree) -> TokenStream {
-        TokenStream::new(vec![(tree, NonJoint)])
-    }
-}
-
-impl From<TokenTree> for TreeAndJoint {
-    fn from(tree: TokenTree) -> TreeAndJoint {
-        (tree, NonJoint)
+        TokenStream::new(vec![tree])
     }
 }
 
 impl iter::FromIterator<TokenTree> for TokenStream {
     fn from_iter<I: IntoIterator<Item = TokenTree>>(iter: I) -> Self {
-        TokenStream::new(iter.into_iter().map(Into::into).collect::<Vec<TreeAndJoint>>())
+        TokenStream::new(iter.into_iter().collect())
     }
 }
 
@@ -209,7 +195,7 @@ impl PartialEq<TokenStream> for TokenStream {
 }
 
 impl TokenStream {
-    pub fn new(streams: Vec<TreeAndJoint>) -> TokenStream {
+    pub fn new(streams: Vec<TokenTree>) -> TokenStream {
         TokenStream(Lrc::new(streams))
     }
 
@@ -224,8 +210,8 @@ impl TokenStream {
     pub fn span(&self) -> Option<Span> {
         match &**self.0 {
             [] => None,
-            [(tt, _)] => Some(tt.span()),
-            [(tt_start, _), .., (tt_end, _)] => Some(tt_start.span().to(tt_end.span())),
+            [tt] => Some(tt.span()),
+            [tt_start, .., tt_end] => Some(tt_start.span().to(tt_end.span())),
         }
     }
 
@@ -290,18 +276,12 @@ impl TokenStream {
 
     pub fn map_enumerated<F: FnMut(usize, TokenTree) -> TokenTree>(self, mut f: F) -> TokenStream {
         TokenStream(Lrc::new(
-            self.0
-                .iter()
-                .enumerate()
-                .map(|(i, (tree, is_joint))| (f(i, tree.clone()), *is_joint))
-                .collect(),
+            self.0.iter().enumerate().map(|(i, tree)| f(i, tree.clone())).collect(),
         ))
     }
 
     pub fn map<F: FnMut(TokenTree) -> TokenTree>(self, mut f: F) -> TokenStream {
-        TokenStream(Lrc::new(
-            self.0.iter().map(|(tree, is_joint)| (f(tree.clone()), *is_joint)).collect(),
-        ))
+        TokenStream(Lrc::new(self.0.iter().map(|tree| f(tree.clone())).collect()))
     }
 }
 
@@ -320,11 +300,11 @@ impl TokenStreamBuilder {
         // If `self` is not empty and the last tree within the last stream is a
         // token tree marked with `Joint`...
         if let Some(TokenStream(ref mut last_stream_lrc)) = self.0.last_mut() {
-            if let Some((TokenTree::Token(last_token), Joint)) = last_stream_lrc.last() {
+            if let Some(TokenTree::Token(last_token)) = last_stream_lrc.last() {
                 // ...and `stream` is not empty and the first tree within it is
                 // a token tree...
                 let TokenStream(ref mut stream_lrc) = stream;
-                if let Some((TokenTree::Token(token), is_joint)) = stream_lrc.first() {
+                if let Some(TokenTree::Token(token)) = stream_lrc.first() {
                     // ...and the two tokens can be glued together...
                     if let Some(glued_tok) = last_token.glue(&token) {
                         // ...then do so, by overwriting the last token
@@ -337,8 +317,7 @@ impl TokenStreamBuilder {
                         // Overwrite the last token tree with the merged
                         // token.
                         let last_vec_mut = Lrc::make_mut(last_stream_lrc);
-                        *last_vec_mut.last_mut().unwrap() =
-                            (TokenTree::Token(glued_tok), *is_joint);
+                        *last_vec_mut.last_mut().unwrap() = TokenTree::Token(glued_tok);
 
                         // Remove the first token tree from `stream`. (This
                         // is almost always the only tree in `stream`.)
@@ -375,23 +354,19 @@ impl Iterator for Cursor {
     type Item = TokenTree;
 
     fn next(&mut self) -> Option<TokenTree> {
-        self.next_with_joint().map(|(tree, _)| tree)
-    }
-}
-
-impl Cursor {
-    fn new(stream: TokenStream) -> Self {
-        Cursor { stream, index: 0 }
-    }
-
-    pub fn next_with_joint(&mut self) -> Option<TreeAndJoint> {
         if self.index < self.stream.len() {
             self.index += 1;
             Some(self.stream.0[self.index - 1].clone())
         } else {
             None
         }
     }
+}
+
+impl Cursor {
+    fn new(stream: TokenStream) -> Self {
+        Cursor { stream, index: 0 }
+    }
 
     pub fn append(&mut self, new_stream: TokenStream) {
         if new_stream.is_empty() {
@@ -404,7 +379,7 @@ impl Cursor {
     }
 
     pub fn look_ahead(&self, n: usize) -> Option<TokenTree> {
-        self.stream.0[self.index..].get(n).map(|(tree, _)| tree.clone())
+        self.stream.0[self.index..].get(n).cloned()
     }
 }
 

compiler/rustc_expand/src/mbe/quoted.rs

@@ -56,31 +56,29 @@ pub(super) fn parse(
         match tree {
             TokenTree::MetaVar(start_sp, ident) if expect_matchers => {
                 let span = match trees.next() {
-                    Some(tokenstream::TokenTree::Token(Token { kind: token::Colon, span })) => {
-                        match trees.next() {
-                            Some(tokenstream::TokenTree::Token(token)) => match token.ident() {
-                                Some((frag, _)) => {
-                                    let span = token.span.with_lo(start_sp.lo());
-                                    let kind = token::NonterminalKind::from_symbol(frag.name)
-                                        .unwrap_or_else(|| {
-                                            let msg = format!(
-                                                "invalid fragment specifier `{}`",
-                                                frag.name
-                                            );
-                                            sess.span_diagnostic
-                                                .struct_span_err(span, &msg)
-                                                .help(VALID_FRAGMENT_NAMES_MSG)
-                                                .emit();
-                                            token::NonterminalKind::Ident
-                                        });
-                                    result.push(TokenTree::MetaVarDecl(span, ident, kind));
-                                    continue;
-                                }
-                                _ => token.span,
-                            },
-                            tree => tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(span),
-                        }
-                    }
+                    Some(tokenstream::TokenTree::Token(Token {
+                        kind: token::Colon, span, ..
+                    })) => match trees.next() {
+                        Some(tokenstream::TokenTree::Token(token)) => match token.ident() {
+                            Some((frag, _)) => {
+                                let span = token.span.with_lo(start_sp.lo());
+                                let kind = token::NonterminalKind::from_symbol(frag.name)
+                                    .unwrap_or_else(|| {
+                                        let msg =
+                                            format!("invalid fragment specifier `{}`", frag.name);
+                                        sess.span_diagnostic
+                                            .struct_span_err(span, &msg)
+                                            .help(VALID_FRAGMENT_NAMES_MSG)
+                                            .emit();
+                                        token::NonterminalKind::Ident
+                                    });
+                                result.push(TokenTree::MetaVarDecl(span, ident, kind));
+                                continue;
+                            }
+                            _ => token.span,
+                        },
+                        tree => tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(span),
+                    },
                     tree => tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(start_sp),
                 };
                 sess.span_diagnostic.struct_span_err(span, "missing fragment specifier").emit();
@@ -119,7 +117,7 @@ fn parse_tree(
     // Depending on what `tree` is, we could be parsing different parts of a macro
     match tree {
         // `tree` is a `$` token. Look at the next token in `trees`
-        tokenstream::TokenTree::Token(Token { kind: token::Dollar, span }) => {
+        tokenstream::TokenTree::Token(Token { kind: token::Dollar, span, .. }) => {
             // FIXME: Handle `None`-delimited groups in a more systematic way
             // during parsing.
             let mut next = outer_trees.next();