refactor(es/parser): Split parser into also-lex/parse-only

.changeset/kind-weeks-study.md

@@ -0,0 +1,5 @@
+---
+swc_common: major
+---
+
+refactor(es/parser): Split parser into also-lex/parse-only

.changeset/twenty-rocks-deny.md

@@ -0,0 +1,6 @@
+---
+swc_ecma_lexer: major
+swc_ecma_parser: major
+---
+
+split token value and token kind

crates/swc_common/src/input.rs

@@ -72,7 +72,7 @@ impl<'a> From<&'a SourceFile> for StringInput<'a> {
     }
 }
 
-impl Input for StringInput<'_> {
+impl<'a> Input<'a> for StringInput<'a> {
     #[inline]
     fn cur(&self) -> Option<char> {
         self.iter.clone().next()
@@ -133,7 +133,7 @@ impl Input for StringInput<'_> {
     }
 
     #[inline]
-    unsafe fn slice(&mut self, start: BytePos, end: BytePos) -> &str {
+    unsafe fn slice(&mut self, start: BytePos, end: BytePos) -> &'a str {
         debug_assert!(start <= end, "Cannot slice {start:?}..{end:?}");
         let s = self.orig;
 
@@ -151,7 +151,7 @@ impl Input for StringInput<'_> {
     }
 
     #[inline]
-    fn uncons_while<F>(&mut self, mut pred: F) -> &str
+    fn uncons_while<F>(&mut self, mut pred: F) -> &'a str
     where
         F: FnMut(char) -> bool,
     {
@@ -237,7 +237,7 @@ impl Input for StringInput<'_> {
     }
 }
 
-pub trait Input: Clone {
+pub trait Input<'a>: Clone {
     fn cur(&self) -> Option<char>;
     fn peek(&self) -> Option<char>;
     fn peek_ahead(&self) -> Option<char>;
@@ -270,11 +270,11 @@ pub trait Input: Clone {
     ///
     /// - start should be less than or equal to end.
     /// - start and end should be in the valid range of input.
-    unsafe fn slice(&mut self, start: BytePos, end: BytePos) -> &str;
+    unsafe fn slice(&mut self, start: BytePos, end: BytePos) -> &'a str;
 
     /// Takes items from stream, testing each one with predicate. returns the
     /// range of items which passed predicate.
-    fn uncons_while<F>(&mut self, f: F) -> &str
+    fn uncons_while<F>(&mut self, f: F) -> &'a str
     where
         F: FnMut(char) -> bool;
 

crates/swc_css_parser/src/lexer/mod.rs

@@ -21,7 +21,7 @@ pub(crate) type LexResult<T> = Result<T, ErrorKind>;
 #[derive(Clone)]
 pub struct Lexer<'a, I>
 where
-    I: Input,
+    I: Input<'a>,
 {
     comments: Option<&'a dyn Comments>,
     pending_leading_comments: Vec<Comment>,
@@ -41,7 +41,7 @@ where
 
 impl<'a, I> Lexer<'a, I>
 where
-    I: Input,
+    I: Input<'a>,
 {
     pub fn new(input: I, comments: Option<&'a dyn Comments>, config: ParserConfig) -> Self {
         let start_pos = input.last_pos();
@@ -65,7 +65,7 @@ where
 
     fn with_buf<F, Ret>(&mut self, op: F) -> LexResult<Ret>
     where
-        F: for<'any> FnOnce(&mut Lexer<I>, &mut String) -> LexResult<Ret>,
+        F: for<'any> FnOnce(&mut Lexer<'a, I>, &mut String) -> LexResult<Ret>,
     {
         let b = self.buf.clone();
         let mut buf = b.borrow_mut();
@@ -77,7 +77,7 @@ where
 
     fn with_sub_buf<F, Ret>(&mut self, op: F) -> LexResult<Ret>
     where
-        F: for<'any> FnOnce(&mut Lexer<I>, &mut String) -> LexResult<Ret>,
+        F: for<'any> FnOnce(&mut Lexer<'a, I>, &mut String) -> LexResult<Ret>,
     {
         let b = self.sub_buf.clone();
         let mut sub_buf = b.borrow_mut();
@@ -89,7 +89,7 @@ where
 
     fn with_buf_and_raw_buf<F, Ret>(&mut self, op: F) -> LexResult<Ret>
     where
-        F: for<'any> FnOnce(&mut Lexer<I>, &mut String, &mut String) -> LexResult<Ret>,
+        F: for<'any> FnOnce(&mut Lexer<'a, I>, &mut String, &mut String) -> LexResult<Ret>,
     {
         let b = self.buf.clone();
         let r = self.raw_buf.clone();
@@ -103,7 +103,7 @@ where
     }
 }
 
-impl<I: Input> Iterator for Lexer<'_, I> {
+impl<'a, I: Input<'a>> Iterator for Lexer<'a, I> {
     type Item = TokenAndSpan;
 
     fn next(&mut self) -> Option<Self::Item> {
@@ -133,9 +133,9 @@ pub struct LexerState {
     pos: BytePos,
 }
 
-impl<I> ParserInput for Lexer<'_, I>
+impl<'a, I> ParserInput for Lexer<'a, I>
 where
-    I: Input,
+    I: Input<'a>,
 {
     type State = LexerState;
 
@@ -185,9 +185,9 @@ where
     }
 }
 
-impl<I> Lexer<'_, I>
+impl<'a, I> Lexer<'a, I>
 where
-    I: Input,
+    I: Input<'a>,
 {
     #[inline(always)]
     fn cur(&mut self) -> Option<char> {

crates/swc_css_parser/src/lib.rs

@@ -41,14 +41,14 @@ where
 ///
 /// If there are syntax errors but if it was recoverable, it will be appended
 /// to `errors`.
-pub fn parse_file<'a, 'b, T>(
+pub fn parse_file<'a, T>(
     fm: &'a SourceFile,
-    comments: Option<&'b dyn Comments>,
+    comments: Option<&'a dyn Comments>,
     config: ParserConfig,
     errors: &mut Vec<Error>,
 ) -> PResult<T>
 where
-    Parser<Lexer<'b, StringInput<'a>>>: Parse<T>,
+    Parser<Lexer<'a, StringInput<'a>>>: Parse<T>,
 {
     parse_string_input(StringInput::from(fm), comments, config, errors)
 }
@@ -57,14 +57,14 @@ where
 ///
 /// If there are syntax errors but if it was recoverable, it will be appended
 /// to `errors`.
-pub fn parse_string_input<'a, 'b, T>(
+pub fn parse_string_input<'a, T>(
     input: StringInput<'a>,
-    comments: Option<&'b dyn Comments>,
+    comments: Option<&'a dyn Comments>,
     config: ParserConfig,
     errors: &mut Vec<Error>,
 ) -> PResult<T>
 where
-    Parser<Lexer<'b, StringInput<'a>>>: Parse<T>,
+    Parser<Lexer<'a, StringInput<'a>>>: Parse<T>,
 {
     let lexer = Lexer::new(input, comments, config);
     let mut parser = Parser::new(lexer, config);

crates/swc_ecma_lexer/Cargo.toml

@@ -61,6 +61,9 @@ swc_ecma_visit = { version = "10.0.0", path = "../swc_ecma_visit" }
 swc_malloc = { version = "1.2.2", path = "../swc_malloc" }
 testing = { version = "11.0.0", path = "../testing" }
 
+[[example]]
+name = "lexer"
+
 [[bench]]
 harness = false
 name    = "lexer"

crates/swc_ecma_parser/examples/lexer.rs → crates/swc_ecma_lexer/examples/lexer.rs

@@ -1,9 +1,10 @@
 use swc_common::{
     errors::{ColorConfig, Handler},
+    input::StringInput,
     sync::Lrc,
     FileName, SourceMap,
 };
-use swc_ecma_parser::{lexer::Lexer, Capturing, Parser, StringInput, Syntax};
+use swc_ecma_lexer::{lexer, lexer::Lexer, Syntax};
 
 fn main() {
     let cm: Lrc<SourceMap> = Default::default();
@@ -19,25 +20,15 @@ fn main() {
         "function foo() {}".into(),
     );
 
-    let lexer = Lexer::new(
+    let l = Lexer::new(
         Syntax::Es(Default::default()),
         Default::default(),
         StringInput::from(&*fm),
         None,
     );
 
-    let capturing = Capturing::new(lexer);
-
-    let mut parser = Parser::new_from(capturing);
-
-    for e in parser.take_errors() {
-        e.into_diagnostic(&handler).emit();
-    }
-
-    let _module = parser
-        .parse_module()
+    let tokens = lexer(l)
         .map_err(|e| e.into_diagnostic(&handler).emit())
-        .expect("Failed to parse module.");
-
-    println!("Tokens: {:?}", parser.input().take());
+        .expect("Failed to lex.");
+    println!("Tokens: {tokens:?}",);
 }

crates/swc_ecma_lexer/src/common/context.rs

@@ -0,0 +1,69 @@
+bitflags::bitflags! {
+  #[derive(Debug, Clone, Copy, Default)]
+  pub struct Context: u32 {
+
+      /// `true` while backtracking
+      const IgnoreError = 1 << 0;
+
+      /// Is in module code?
+      const Module = 1 << 1;
+      const CanBeModule = 1 << 2;
+      const Strict = 1 << 3;
+
+      const ForLoopInit = 1 << 4;
+      const ForAwaitLoopInit = 1 << 5;
+
+      const IncludeInExpr = 1 << 6;
+      /// If true, await expression is parsed, and "await" is treated as a
+      /// keyword.
+      const InAsync = 1 << 7;
+      /// If true, yield expression is parsed, and "yield" is treated as a
+      /// keyword.
+      const InGenerator = 1 << 8;
+
+      /// If true, await is treated as a keyword.
+      const InStaticBlock = 1 << 9;
+
+      const IsContinueAllowed = 1 << 10;
+      const IsBreakAllowed = 1 << 11;
+
+      const InType = 1 << 12;
+      /// Typescript extension.
+      const ShouldNotLexLtOrGtAsType = 1 << 13;
+      /// Typescript extension.
+      const InDeclare = 1 << 14;
+
+      /// If true, `:` should not be treated as a type annotation.
+      const InCondExpr = 1 << 15;
+      const WillExpectColonForCond = 1 << 16;
+
+      const InClass = 1 << 17;
+
+      const InClassField = 1 << 18;
+
+      const InFunction = 1 << 19;
+
+      /// This indicates current scope or the scope out of arrow function is
+      /// function declaration or function expression or not.
+      const InsideNonArrowFunctionScope = 1 << 20;
+
+      const InParameters = 1 << 21;
+
+      const HasSuperClass = 1 << 22;
+
+      const InPropertyName = 1 << 23;
+
+      const InForcedJsxContext = 1 << 24;
+
+      // If true, allow super.x and super[x]
+      const AllowDirectSuper = 1 << 25;
+
+      const IgnoreElseClause = 1 << 26;
+
+      const DisallowConditionalTypes = 1 << 27;
+
+      const AllowUsingDecl = 1 << 28;
+
+      const TopLevel = 1 << 29;
+  }
+}

crates/swc_ecma_lexer/src/common/input.rs

@@ -0,0 +1,48 @@
+use swc_common::BytePos;
+use swc_ecma_ast::EsVersion;
+
+use super::{context::Context, syntax::Syntax};
+use crate::{error::Error, lexer};
+
+/// Clone should be cheap if you are parsing typescript because typescript
+/// syntax requires backtracking.
+pub trait Tokens<TokenAndSpan>: Clone + Iterator<Item = TokenAndSpan> {
+    fn set_ctx(&mut self, ctx: Context);
+    fn ctx(&self) -> Context;
+    fn syntax(&self) -> Syntax;
+    fn target(&self) -> EsVersion;
+
+    fn start_pos(&self) -> BytePos {
+        BytePos(0)
+    }
+
+    fn set_expr_allowed(&mut self, allow: bool);
+    fn set_next_regexp(&mut self, start: Option<BytePos>);
+
+    fn token_context(&self) -> &lexer::TokenContexts;
+    fn token_context_mut(&mut self) -> &mut lexer::TokenContexts;
+    fn set_token_context(&mut self, _c: lexer::TokenContexts);
+
+    /// Implementors should use Rc<RefCell<Vec<Error>>>.
+    ///
+    /// It is required because parser should backtrack while parsing typescript
+    /// code.
+    fn add_error(&self, error: Error);
+
+    /// Add an error which is valid syntax in script mode.
+    ///
+    /// This errors should be dropped if it's not a module.
+    ///
+    /// Implementor should check for if [Context].module, and buffer errors if
+    /// module is false. Also, implementors should move errors to the error
+    /// buffer on set_ctx if the parser mode become module mode.
+    fn add_module_mode_error(&self, error: Error);
+
+    fn end_pos(&self) -> BytePos;
+
+    fn take_errors(&mut self) -> Vec<Error>;
+
+    /// If the program was parsed as a script, this contains the module
+    /// errors should the program be identified as a module in the future.
+    fn take_script_module_errors(&mut self) -> Vec<Error>;
+}