perf(es/parser): Defne Token as TokenType + TokenValue

crates/swc_ecma_lexer/src/error.rs

@@ -8,7 +8,7 @@ use swc_common::{
     Span, Spanned,
 };
 
-use crate::token::Token;
+use crate::token::TokenType;
 
 /// Note: this struct is 8 bytes.
 #[derive(Debug, Clone, PartialEq)]
@@ -126,7 +126,7 @@ pub enum SyntaxError {
     },
     ReservedWordInImport,
     AssignProperty,
-    Expected(&'static Token, String),
+    Expected(&'static TokenType, String),
     ExpectedSemiForExprStmt {
         expr: Span,
     },

crates/swc_ecma_lexer/src/input.rs

@@ -309,7 +309,7 @@ impl<I: Tokens> Buffer<I> {
         }
     }
 
-    pub fn store(&mut self, token: Token) {
+    pub fn store(&mut self, token: TokenType) {
         debug_assert!(self.next.is_none());
         debug_assert!(self.cur.is_none());
         let span = self.prev_span;
@@ -322,7 +322,7 @@ impl<I: Tokens> Buffer<I> {
     }
 
     #[allow(dead_code)]
-    pub fn cur_debug(&self) -> Option<&Token> {
+    pub fn cur_debug(&self) -> Option<&TokenType> {
         self.cur.as_ref().map(|it| &it.token)
     }
 
@@ -336,7 +336,7 @@ impl<I: Tokens> Buffer<I> {
     }
 
     /// Returns current token.
-    pub fn bump(&mut self) -> Token {
+    pub fn bump(&mut self) -> TokenType {
         let prev = match self.cur.take() {
             Some(t) => t,
             None => unsafe {
@@ -355,7 +355,7 @@ impl<I: Tokens> Buffer<I> {
         self.cur.is_some()
     }
 
-    pub fn peek(&mut self) -> Option<&Token> {
+    pub fn peek(&mut self) -> Option<&TokenType> {
         debug_assert!(
             self.cur.is_some(),
             "parser should not call peek() without knowing current token"
@@ -392,7 +392,7 @@ impl<I: Tokens> Buffer<I> {
 
     /// Get current token. Returns `None` only on eof.
     #[inline]
-    pub fn cur(&mut self) -> Option<&Token> {
+    pub fn cur(&mut self) -> Option<&TokenType> {
         if self.cur.is_none() {
             // If we have peeked a token, take it instead of calling lexer.next()
             self.cur = self.next.take().or_else(|| self.iter.next());
@@ -464,15 +464,15 @@ impl<I: Tokens> Buffer<I> {
     }
 
     #[inline]
-    pub fn is(&mut self, expected: &Token) -> bool {
+    pub fn is(&mut self, expected: &TokenType) -> bool {
         match self.cur() {
             Some(t) => *expected == *t,
             _ => false,
         }
     }
 
     #[inline]
-    pub fn eat(&mut self, expected: &Token) -> bool {
+    pub fn eat(&mut self, expected: &TokenType) -> bool {
         let v = self.is(expected);
         if v {
             self.bump();

crates/swc_ecma_lexer/src/lexer/jsx.rs

@@ -2,6 +2,7 @@ use either::Either;
 use smartstring::{LazyCompact, SmartString};
 
 use super::*;
+use crate::token::Token;
 
 impl Lexer<'_> {
     pub(super) fn read_jsx_token(&mut self) -> LexResult<Option<Token>> {
@@ -38,7 +39,7 @@ impl Lexer<'_> {
                                 // Safety: cur() was Some('<')
                                 self.input.bump();
                             }
-                            return Ok(Some(Token::JSXTagStart));
+                            return Ok(Some(TokenType::JSXTagStart));
                         }
                         return self.read_token();
                     }
@@ -66,7 +67,7 @@ impl Lexer<'_> {
                         self.atoms.atom(s)
                     };
 
-                    return Ok(Some(Token::JSXText { raw, value }));
+                    return Ok(Some(TokenType::JSXText { raw, value }));
                 }
                 '>' => {
                     self.emit_error(
@@ -348,7 +349,7 @@ impl Lexer<'_> {
             self.input.slice(start, end)
         };
 
-        Ok(Token::Str {
+        Ok(TokenType::Str {
             value,
             raw: self.atoms.atom(raw),
         })
@@ -375,7 +376,7 @@ impl Lexer<'_> {
             }
         });
 
-        Ok(Token::JSXName {
+        Ok(TokenType::JSXName {
             name: self.atoms.atom(slice),
         })
     }

crates/swc_ecma_lexer/src/lexer/mod.rs

@@ -22,7 +22,7 @@ use self::{
 use crate::{
     error::{Error, SyntaxError},
     tok,
-    token::{BinOpToken, IdentLike, Token, Word},
+    token::{BinOpToken, IdentLike, TokenType, Word},
     Context, Syntax,
 };
 
@@ -231,7 +231,7 @@ impl<'a> Lexer<'a> {
             !self.input.is_at_start() || self.cur() != Some('!'),
             "#! should have already been handled by read_shebang()"
         );
-        Ok(Some(Token::Hash))
+        Ok(Some(TokenType::Hash))
     }
 
     /// Read a token given `.`.
@@ -252,8 +252,8 @@ impl<'a> Lexer<'a> {
         };
         if next.is_ascii_digit() {
             return self.read_number(true).map(|v| match v {
-                Left((value, raw)) => Token::Num { value, raw },
-                Right((value, raw)) => Token::BigInt { value, raw },
+                Left((value, raw)) => TokenType::Num { value, raw },
+                Right((value, raw)) => TokenType::BigInt { value, raw },
             });
         }
 
@@ -334,15 +334,15 @@ impl<'a> Lexer<'a> {
             Some('b') | Some('B') => self.read_radix_number::<2>(),
             _ => {
                 return self.read_number(false).map(|v| match v {
-                    Left((value, raw)) => Token::Num { value, raw },
-                    Right((value, raw)) => Token::BigInt { value, raw },
+                    Left((value, raw)) => TokenType::Num { value, raw },
+                    Right((value, raw)) => TokenType::BigInt { value, raw },
                 });
             }
         };
 
         bigint.map(|v| match v {
-            Left((value, raw)) => Token::Num { value, raw },
-            Right((value, raw)) => Token::BigInt { value, raw },
+            Left((value, raw)) => TokenType::Num { value, raw },
+            Right((value, raw)) => TokenType::BigInt { value, raw },
         })
     }
 
@@ -366,7 +366,7 @@ impl<'a> Lexer<'a> {
 
         // '|=', '&='
         if self.input.eat_byte(b'=') {
-            return Ok(Token::AssignOp(match token {
+            return Ok(TokenType::AssignOp(match token {
                 BinOpToken::BitAnd => AssignOp::BitAndAssign,
                 BinOpToken::BitOr => AssignOp::BitOrAssign,
                 _ => unreachable!(),
@@ -385,7 +385,7 @@ impl<'a> Lexer<'a> {
                     // Safety: cur() is Some('=')
                     self.input.bump();
                 }
-                return Ok(Token::AssignOp(match token {
+                return Ok(TokenType::AssignOp(match token {
                     BinOpToken::BitAnd => op!("&&="),
                     BinOpToken::BitOr => op!("||="),
                     _ => unreachable!(),
@@ -402,14 +402,14 @@ impl<'a> Lexer<'a> {
                 return self.error_span(span, SyntaxError::TS1185);
             }
 
-            return Ok(Token::BinOp(match token {
+            return Ok(TokenType::BinOp(match token {
                 BinOpToken::BitAnd => BinOpToken::LogicalAnd,
                 BinOpToken::BitOr => BinOpToken::LogicalOr,
                 _ => unreachable!(),
             }));
         }
 
-        Ok(Token::BinOp(token))
+        Ok(TokenType::BinOp(token))
     }
 
     /// Read a token given `*` or `%`.
@@ -423,21 +423,21 @@ impl<'a> Lexer<'a> {
             self.input.bump();
         }
         let mut token = if is_mul {
-            Token::BinOp(BinOpToken::Mul)
+            TokenType::BinOp(BinOpToken::Mul)
         } else {
-            Token::BinOp(BinOpToken::Mod)
+            TokenType::BinOp(BinOpToken::Mod)
         };
 
         // check for **
         if is_mul && self.input.eat_byte(b'*') {
-            token = Token::BinOp(BinOpToken::Exp)
+            token = TokenType::BinOp(BinOpToken::Exp)
         }
 
         if self.input.eat_byte(b'=') {
             token = match token {
-                Token::BinOp(BinOpToken::Mul) => Token::AssignOp(AssignOp::MulAssign),
-                Token::BinOp(BinOpToken::Mod) => Token::AssignOp(AssignOp::ModAssign),
-                Token::BinOp(BinOpToken::Exp) => Token::AssignOp(AssignOp::ExpAssign),
+                TokenType::BinOp(BinOpToken::Mul) => TokenType::AssignOp(AssignOp::MulAssign),
+                TokenType::BinOp(BinOpToken::Mod) => TokenType::AssignOp(AssignOp::ModAssign),
+                TokenType::BinOp(BinOpToken::Exp) => TokenType::AssignOp(AssignOp::ExpAssign),
                 _ => unreachable!(),
             }
         }
@@ -596,18 +596,18 @@ impl<'a> Lexer<'a> {
             }
 
             if C == b'+' {
-                Token::PlusPlus
+                TokenType::PlusPlus
             } else {
-                Token::MinusMinus
+                TokenType::MinusMinus
             }
         } else if self.input.eat_byte(b'=') {
-            Token::AssignOp(if C == b'+' {
+            TokenType::AssignOp(if C == b'+' {
                 AssignOp::AddAssign
             } else {
                 AssignOp::SubAssign
             })
         } else {
-            Token::BinOp(if C == b'+' {
+            TokenType::BinOp(if C == b'+' {
                 BinOpToken::Add
             } else {
                 BinOpToken::Sub
@@ -629,7 +629,7 @@ impl<'a> Lexer<'a> {
 
             if self.input.eat_byte(b'=') {
                 if C == b'!' {
-                    Token::BinOp(BinOpToken::NotEqEq)
+                    TokenType::BinOp(BinOpToken::NotEqEq)
                 } else {
                     // =======
                     //    ^
@@ -640,21 +640,21 @@ impl<'a> Lexer<'a> {
                         return self.read_token();
                     }
 
-                    Token::BinOp(BinOpToken::EqEqEq)
+                    TokenType::BinOp(BinOpToken::EqEqEq)
                 }
             } else if C == b'!' {
-                Token::BinOp(BinOpToken::NotEq)
+                TokenType::BinOp(BinOpToken::NotEq)
             } else {
-                Token::BinOp(BinOpToken::EqEq)
+                TokenType::BinOp(BinOpToken::EqEq)
             }
         } else if C == b'=' && self.input.eat_byte(b'>') {
             // "=>"
 
-            Token::Arrow
+            TokenType::Arrow
         } else if C == b'!' {
-            Token::Bang
+            TokenType::Bang
         } else {
-            Token::AssignOp(AssignOp::Assign)
+            TokenType::AssignOp(AssignOp::Assign)
         }))
     }
 }
@@ -725,15 +725,15 @@ impl Lexer<'_> {
 
         let token = if self.eat(b'=') {
             match op {
-                BinOpToken::Lt => Token::BinOp(BinOpToken::LtEq),
-                BinOpToken::Gt => Token::BinOp(BinOpToken::GtEq),
-                BinOpToken::LShift => Token::AssignOp(AssignOp::LShiftAssign),
-                BinOpToken::RShift => Token::AssignOp(AssignOp::RShiftAssign),
-                BinOpToken::ZeroFillRShift => Token::AssignOp(AssignOp::ZeroFillRShiftAssign),
+                BinOpToken::Lt => TokenType::BinOp(BinOpToken::LtEq),
+                BinOpToken::Gt => TokenType::BinOp(BinOpToken::GtEq),
+                BinOpToken::LShift => TokenType::AssignOp(AssignOp::LShiftAssign),
+                BinOpToken::RShift => TokenType::AssignOp(AssignOp::RShiftAssign),
+                BinOpToken::ZeroFillRShift => TokenType::AssignOp(AssignOp::ZeroFillRShiftAssign),
                 _ => unreachable!(),
             }
         } else {
-            Token::BinOp(op)
+            TokenType::BinOp(op)
         };
 
         // All conflict markers consist of the same character repeated seven times.
@@ -799,7 +799,7 @@ impl Lexer<'_> {
                 SyntaxError::EscapeInReservedWord { word: word.into() },
             )?
         } else {
-            Ok(Some(Token::Word(word)))
+            Ok(Some(TokenType::Word(word)))
         }
     }
 
@@ -1076,7 +1076,7 @@ impl Lexer<'_> {
                         };
                         let raw = l.atoms.atom(raw);
 
-                        return Ok(Token::Str { value, raw });
+                        return Ok(TokenType::Str { value, raw });
                     }
 
                     if c == b'\\' {
@@ -1146,7 +1146,7 @@ impl Lexer<'_> {
                 // `self.input`
                 l.input.slice(start, end)
             };
-            Ok(Token::Str {
+            Ok(TokenType::Str {
                 value: l.atoms.atom(&*buf),
                 raw: l.atoms.atom(raw),
             })
@@ -1225,7 +1225,7 @@ impl Lexer<'_> {
         .map(|(value, _)| value)
         .unwrap_or_default();
 
-        Ok(Token::Regex(content, flags))
+        Ok(TokenType::Regex(content, flags))
     }
 
     #[cold]
@@ -1299,7 +1299,7 @@ impl Lexer<'_> {
                     // from `self.input`
                     self.input.slice(raw_slice_start, end)
                 };
-                return Ok(Token::Template {
+                return Ok(TokenType::Template {
                     cooked,
                     raw: self.atoms.atom(raw),
                 });

crates/swc_ecma_lexer/src/lexer/number.rs

@@ -798,7 +798,7 @@ mod tests {
 
                 let token = vec.into_iter().next().unwrap();
                 let value = match token {
-                    Token::Num { value, .. } => value,
+                    TokenType::Num { value, .. } => value,
                     _ => {
                         panic!("expected num token in test")
                     }
@@ -807,7 +807,7 @@ mod tests {
                 assert_eq!(expected, value);
             } else if let Ok(vec) = vec {
                 assert_ne!(
-                    vec![Token::Num {
+                    vec![TokenType::Num {
                         value: expected,
                         raw: expected.to_string().into()
                     }],