From b526da947c29ea050b12869e7c00727ac06d0554 Mon Sep 17 00:00:00 2001 From: Boshen <1430279+Boshen@users.noreply.github.com> Date: Sun, 11 May 2025 03:57:33 +0000 Subject: [PATCH] refactor(parser): make `Token` fields private (#10936) The goal of this PR is to reduce the diff for #10933 --- crates/oxc_parser/src/cursor.rs | 26 +- crates/oxc_parser/src/js/arrow.rs | 14 +- crates/oxc_parser/src/js/class.rs | 2 +- crates/oxc_parser/src/js/declaration.rs | 2 +- crates/oxc_parser/src/js/expression.rs | 34 +-- crates/oxc_parser/src/js/function.rs | 4 +- crates/oxc_parser/src/js/module.rs | 4 +- crates/oxc_parser/src/js/object.rs | 2 +- crates/oxc_parser/src/js/statement.rs | 10 +- crates/oxc_parser/src/lexer/byte_handlers.rs | 4 +- crates/oxc_parser/src/lexer/comment.rs | 24 +- crates/oxc_parser/src/lexer/jsx.rs | 2 +- crates/oxc_parser/src/lexer/kind.rs | 1 + crates/oxc_parser/src/lexer/mod.rs | 10 +- crates/oxc_parser/src/lexer/punctuation.rs | 2 +- crates/oxc_parser/src/lexer/regex.rs | 14 +- crates/oxc_parser/src/lexer/string.rs | 14 +- crates/oxc_parser/src/lexer/template.rs | 12 +- crates/oxc_parser/src/lexer/token.rs | 266 +++++++++++++++++- crates/oxc_parser/src/lexer/trivia_builder.rs | 4 +- crates/oxc_parser/src/lexer/typescript.rs | 4 +- crates/oxc_parser/src/lexer/unicode.rs | 12 +- crates/oxc_parser/src/lexer/whitespace.rs | 2 +- crates/oxc_parser/src/modifiers.rs | 10 +- crates/oxc_parser/src/ts/statement.rs | 22 +- crates/oxc_parser/src/ts/types.rs | 16 +- tasks/benchmark/benches/lexer.rs | 4 +- 27 files changed, 385 insertions(+), 136 deletions(-) diff --git a/crates/oxc_parser/src/cursor.rs b/crates/oxc_parser/src/cursor.rs index 61fa3ba69b438..eb0b046372172 100644 --- a/crates/oxc_parser/src/cursor.rs +++ b/crates/oxc_parser/src/cursor.rs @@ -22,7 +22,7 @@ pub struct ParserCheckpoint<'a> { impl<'a> ParserImpl<'a> { #[inline] pub(crate) fn start_span(&self) -> u32 { - self.token.start + self.token.start() } #[inline] @@ -39,7 +39,7 @@ impl<'a> ParserImpl<'a> { /// Get current Kind #[inline] pub(crate) fn cur_kind(&self) -> Kind { - self.token.kind + self.token.kind() } /// Get current source text @@ -58,7 +58,7 @@ impl<'a> ParserImpl<'a> { /// Get current template string pub(crate) fn cur_template_string(&self) -> Option<&'a str> { - self.lexer.get_template_string(self.token.start) + self.lexer.get_template_string(self.token.start()) } /// Peek next token, returns EOF for final peek @@ -70,13 +70,13 @@ impl<'a> ParserImpl<'a> { /// Peek next kind, returns EOF for final peek #[inline] pub(crate) fn peek_kind(&mut self) -> Kind { - self.peek_token().kind + self.peek_token().kind() } /// Peek at kind #[inline] pub(crate) fn peek_at(&mut self, kind: Kind) -> bool { - self.peek_token().kind == kind + self.peek_token().kind() == kind } /// Peek nth token @@ -91,13 +91,13 @@ impl<'a> ParserImpl<'a> { /// Peek at nth kind #[inline] pub(crate) fn nth_at(&mut self, n: u8, kind: Kind) -> bool { - self.nth(n).kind == kind + self.nth(n).kind() == kind } /// Peek nth kind #[inline] pub(crate) fn nth_kind(&mut self, n: u8) -> Kind { - self.nth(n).kind + self.nth(n).kind() } /// Checks if the current index has token `Kind` @@ -121,7 +121,7 @@ impl<'a> ParserImpl<'a> { /// Checks if the current token is escaped if it is a keyword fn advance(&mut self, kind: Kind) { self.test_escaped_keyword(kind); - self.prev_token_end = self.token.end; + self.prev_token_end = self.token.end(); self.token = self.lexer.next_token(); } @@ -129,7 +129,7 @@ impl<'a> ParserImpl<'a> { /// Checks if the current token is escaped if it is a keyword fn advance_for_jsx_child(&mut self, kind: Kind) { self.test_escaped_keyword(kind); - self.prev_token_end = self.token.end; + self.prev_token_end = self.token.end(); self.token = self.lexer.next_jsx_child(); } @@ -182,7 +182,7 @@ impl<'a> ParserImpl<'a> { if kind == Kind::Semicolon { return true; } - kind == Kind::RCurly || kind.is_eof() || self.cur_token().is_on_new_line + kind == Kind::RCurly || kind.is_eof() || self.cur_token().is_on_new_line() } /// # Errors @@ -242,7 +242,7 @@ impl<'a> ParserImpl<'a> { let kind = self.cur_kind(); if kind == Kind::RAngle { self.token = self.lexer.next_right_angle(); - self.token.kind + self.token.kind() } else { kind } @@ -252,7 +252,7 @@ impl<'a> ParserImpl<'a> { let kind = self.cur_kind(); if matches!(kind, Kind::ShiftLeft | Kind::ShiftLeftEq | Kind::LtEq) { self.token = self.lexer.re_lex_as_typescript_l_angle(kind); - self.token.kind + self.token.kind() } else { kind } @@ -262,7 +262,7 @@ impl<'a> ParserImpl<'a> { let kind = self.cur_kind(); if matches!(kind, Kind::ShiftRight | Kind::ShiftRight3) { self.token = self.lexer.re_lex_as_typescript_r_angle(kind); - self.token.kind + self.token.kind() } else { kind } diff --git a/crates/oxc_parser/src/js/arrow.rs b/crates/oxc_parser/src/js/arrow.rs index 5457be3739be1..670cf07d7cd9d 100644 --- a/crates/oxc_parser/src/js/arrow.rs +++ b/crates/oxc_parser/src/js/arrow.rs @@ -65,8 +65,8 @@ impl<'a> ParserImpl<'a> { if self.at(Kind::Async) { let second_token = self.peek_token(); - let second = second_token.kind; - if second_token.is_on_new_line { + let second = second_token.kind(); + if second_token.is_on_new_line() { return Tristate::False; } if second != Kind::LParen && second != Kind::LAngle { @@ -191,10 +191,10 @@ impl<'a> ParserImpl<'a> { fn is_un_parenthesized_async_arrow_function_worker(&mut self) -> Tristate { if self.at(Kind::Async) { let first_token = self.peek_token(); - let first = first_token.kind; + let first = first_token.kind(); // If the "async" is followed by "=>" token then it is not a beginning of an async arrow-function // but instead a simple arrow-function which will be parsed inside "parseAssignmentExpressionOrHigher" - if first_token.is_on_new_line || first == Kind::Arrow { + if first_token.is_on_new_line() || first == Kind::Arrow { return Tristate::False; } // Check for un-parenthesized AsyncArrowFunction @@ -240,7 +240,7 @@ impl<'a> ParserImpl<'a> { self.ctx = self.ctx.and_await(has_await); - if self.cur_token().is_on_new_line { + if self.cur_token().is_on_new_line() { self.error(diagnostics::lineterminator_before_arrow(self.cur_token().span())); } @@ -281,7 +281,7 @@ impl<'a> ParserImpl<'a> { self.ctx = self.ctx.and_await(has_await); - if self.cur_token().is_on_new_line { + if self.cur_token().is_on_new_line() { self.error(diagnostics::lineterminator_before_arrow(self.cur_token().span())); } @@ -345,7 +345,7 @@ impl<'a> ParserImpl<'a> { &mut self, allow_return_type_in_arrow_function: bool, ) -> Option> { - let pos = self.cur_token().start; + let pos = self.cur_token().start(); if self.state.not_parenthesized_arrow.contains(&pos) { return None; } diff --git a/crates/oxc_parser/src/js/class.rs b/crates/oxc_parser/src/js/class.rs index f58622d8f5849..ac0bf162db1cb 100644 --- a/crates/oxc_parser/src/js/class.rs +++ b/crates/oxc_parser/src/js/class.rs @@ -227,7 +227,7 @@ impl<'a> ParserImpl<'a> { // async ... if key_name.is_none() && self.at(Kind::Async) && !self.peek_at(Kind::Question) { - if !self.peek_token().is_on_new_line + if !self.peek_token().is_on_new_line() && (self.peek_kind().is_class_element_name_start() || self.peek_at(Kind::Star)) { self.bump(Kind::Async); diff --git a/crates/oxc_parser/src/js/declaration.rs b/crates/oxc_parser/src/js/declaration.rs index 8936c55be8170..7c8a86e2c451f 100644 --- a/crates/oxc_parser/src/js/declaration.rs +++ b/crates/oxc_parser/src/js/declaration.rs @@ -95,7 +95,7 @@ impl<'a> ParserImpl<'a> { let mut definite = false; if binding_kind.is_binding_identifier() && self.at(Kind::Bang) - && !self.cur_token().is_on_new_line + && !self.cur_token().is_on_new_line() { self.eat(Kind::Bang); definite = true; diff --git a/crates/oxc_parser/src/js/expression.rs b/crates/oxc_parser/src/js/expression.rs index ebdaf5e155050..c8ab5ca3ca686 100644 --- a/crates/oxc_parser/src/js/expression.rs +++ b/crates/oxc_parser/src/js/expression.rs @@ -290,9 +290,9 @@ impl<'a> ParserImpl<'a> { let span = self.start_span(); let token = self.cur_token(); let src = self.cur_src(); - let value = match token.kind { + let value = match token.kind() { Kind::Decimal | Kind::Binary | Kind::Octal | Kind::Hex => { - parse_int(src, token.kind, token.has_separator()) + parse_int(src, token.kind(), token.has_separator()) } Kind::Float | Kind::PositiveExponential | Kind::NegativeExponential => { parse_float(src, token.has_separator()) @@ -303,7 +303,7 @@ impl<'a> ParserImpl<'a> { self.set_fatal_error(diagnostics::invalid_number(err, token.span())); 0.0 // Dummy value }); - let base = match token.kind { + let base = match token.kind() { Kind::Decimal => NumberBase::Decimal, Kind::Float => NumberBase::Float, Kind::Binary => NumberBase::Binary, @@ -334,7 +334,7 @@ impl<'a> ParserImpl<'a> { let token = self.cur_token(); let raw = self.cur_src(); let src = raw.strip_suffix('n').unwrap(); - let _value = parse_big_int(src, token.kind, token.has_separator()) + let _value = parse_big_int(src, token.kind(), token.has_separator()) .map_err(|err| diagnostics::invalid_number(err, token.span())); self.bump_any(); self.ast.big_int_literal(self.end_span(span), raw, base) @@ -346,10 +346,10 @@ impl<'a> ParserImpl<'a> { if !self.lexer.errors.is_empty() { return self.unexpected(); } - let pattern_start = self.cur_token().start + 1; // +1 to exclude left `/` + let pattern_start = self.cur_token().start() + 1; // +1 to exclude left `/` let pattern_text = &self.source_text[pattern_start as usize..pattern_end as usize]; let flags_start = pattern_end + 1; // +1 to include right `/` - let flags_text = &self.source_text[flags_start as usize..self.cur_token().end as usize]; + let flags_text = &self.source_text[flags_start as usize..self.cur_token().end() as usize]; let raw = self.cur_src(); self.bump_any(); @@ -406,7 +406,7 @@ impl<'a> ParserImpl<'a> { } let value = self.cur_string(); let span = self.start_span(); - let lone_surrogates = self.cur_token().lone_surrogates; + let lone_surrogates = self.cur_token().lone_surrogates(); self.bump_any(); let span = self.end_span(span); // SAFETY: @@ -547,13 +547,13 @@ impl<'a> ParserImpl<'a> { // Also replace `\r` with `\n` in `raw`. // If contains `\r`, then `escaped` must be `true` (because `\r` needs unescaping), // so we can skip searching for `\r` in common case where contains no escapes. - let (cooked, lone_surrogates) = if self.cur_token().escaped { + let (cooked, lone_surrogates) = if self.cur_token().escaped() { // `cooked = None` when template literal has invalid escape sequence let cooked = self.cur_template_string().map(Atom::from); if cooked.is_some() && raw.contains('\r') { raw = self.ast.atom(&raw.cow_replace("\r\n", "\n").cow_replace('\r', "\n")); } - (cooked, self.cur_token().lone_surrogates) + (cooked, self.cur_token().lone_surrogates()) } else { (Some(raw), false) }; @@ -774,7 +774,7 @@ impl<'a> ParserImpl<'a> { } if !question_dot { - if self.at(Kind::Bang) && !self.cur_token().is_on_new_line && self.is_ts { + if self.at(Kind::Bang) && !self.cur_token().is_on_new_line() && self.is_ts { self.bump_any(); lhs = self.ast.expression_ts_non_null(self.end_span(lhs_span), lhs); continue; @@ -1021,7 +1021,7 @@ impl<'a> ParserImpl<'a> { let span = self.start_span(); let lhs = self.parse_lhs_expression_or_higher(); // ++ -- postfix update expressions - if self.cur_kind().is_update_operator() && !self.cur_token().is_on_new_line { + if self.cur_kind().is_update_operator() && !self.cur_token().is_on_new_line() { let operator = map_update_operator(self.cur_kind()); self.bump_any(); let lhs = SimpleAssignmentTarget::cover(lhs, self); @@ -1131,7 +1131,7 @@ impl<'a> ParserImpl<'a> { } if self.is_ts && matches!(kind, Kind::As | Kind::Satisfies) { - if self.cur_token().is_on_new_line { + if self.cur_token().is_on_new_line() { break; } self.bump_any(); @@ -1396,7 +1396,7 @@ impl<'a> ParserImpl<'a> { if self.at(Kind::Await) { let peek_token = self.peek_token(); // Allow arrow expression `await => {}` - if peek_token.kind == Kind::Arrow { + if peek_token.kind() == Kind::Arrow { return false; } if self.ctx.has_await() { @@ -1405,13 +1405,13 @@ impl<'a> ParserImpl<'a> { // The following expressions are ambiguous // await + 0, await - 0, await ( 0 ), await [ 0 ], await / 0 /u, await ``, await of [] if matches!( - peek_token.kind, + peek_token.kind(), Kind::Of | Kind::LParen | Kind::LBrack | Kind::Slash | Kind::RegExp ) { return false; } - return !peek_token.is_on_new_line && peek_token.kind.is_after_await_or_yield(); + return !peek_token.is_on_new_line() && peek_token.kind().is_after_await_or_yield(); } false } @@ -1420,13 +1420,13 @@ impl<'a> ParserImpl<'a> { if self.at(Kind::Yield) { let peek_token = self.peek_token(); // Allow arrow expression `yield => {}` - if peek_token.kind == Kind::Arrow { + if peek_token.kind() == Kind::Arrow { return false; } if self.ctx.has_yield() { return true; } - return !peek_token.is_on_new_line && peek_token.kind.is_after_await_or_yield(); + return !peek_token.is_on_new_line() && peek_token.kind().is_after_await_or_yield(); } false } diff --git a/crates/oxc_parser/src/js/function.rs b/crates/oxc_parser/src/js/function.rs index 519378ae576e5..274e2a936b53a 100644 --- a/crates/oxc_parser/src/js/function.rs +++ b/crates/oxc_parser/src/js/function.rs @@ -24,7 +24,7 @@ impl<'a> ParserImpl<'a> { self.at(Kind::Function) || self.at(Kind::Async) && self.peek_at(Kind::Function) - && !self.peek_token().is_on_new_line + && !self.peek_token().is_on_new_line() } pub(crate) fn parse_function_body(&mut self) -> Box<'a, FunctionBody<'a>> { @@ -303,7 +303,7 @@ impl<'a> ParserImpl<'a> { let mut delegate = false; let mut argument = None; - if !self.cur_token().is_on_new_line { + if !self.cur_token().is_on_new_line() { delegate = self.eat(Kind::Star); let not_assignment_expr = matches!( self.cur_kind(), diff --git a/crates/oxc_parser/src/js/module.rs b/crates/oxc_parser/src/js/module.rs index 14b2bb1ef6a46..7f355ed78048c 100644 --- a/crates/oxc_parser/src/js/module.rs +++ b/crates/oxc_parser/src/js/module.rs @@ -172,7 +172,7 @@ impl<'a> ParserImpl<'a> { /// [Import Attributes](https://tc39.es/proposal-import-attributes) fn parse_import_attributes(&mut self) -> Option> { let attributes_keyword = match self.cur_kind() { - Kind::Assert if !self.cur_token().is_on_new_line => self.parse_identifier_name(), + Kind::Assert if !self.cur_token().is_on_new_line() => self.parse_identifier_name(), Kind::With => self.parse_identifier_name(), _ => { return None; @@ -403,7 +403,7 @@ impl<'a> ParserImpl<'a> { self.parse_class_declaration(decl_span, &modifiers), ) } - _ if self.at(Kind::Interface) && !self.peek_token().is_on_new_line && self.is_ts => { + _ if self.at(Kind::Interface) && !self.peek_token().is_on_new_line() && self.is_ts => { let decl = self.parse_ts_interface_declaration(decl_span, &Modifiers::empty()); match decl { Declaration::TSInterfaceDeclaration(decl) => { diff --git a/crates/oxc_parser/src/js/object.rs b/crates/oxc_parser/src/js/object.rs index 6704442c62509..25c955ac776bc 100644 --- a/crates/oxc_parser/src/js/object.rs +++ b/crates/oxc_parser/src/js/object.rs @@ -46,7 +46,7 @@ impl<'a> ParserImpl<'a> { // AsyncGeneratorMethod Kind::Async if (class_element_name || peek_kind == Kind::Star) - && !self.peek_token().is_on_new_line => + && !self.peek_token().is_on_new_line() => { self.parse_property_definition_method() } diff --git a/crates/oxc_parser/src/js/statement.rs b/crates/oxc_parser/src/js/statement.rs index d42c61f2ed561..ab43840ca4b9c 100644 --- a/crates/oxc_parser/src/js/statement.rs +++ b/crates/oxc_parser/src/js/statement.rs @@ -117,7 +117,7 @@ impl<'a> ParserImpl<'a> { Kind::Async if { let peek_token = self.peek_token(); - peek_token.kind == Kind::Function && !peek_token.is_on_new_line + peek_token.kind() == Kind::Function && !peek_token.is_on_new_line() } => { self.parse_function_declaration(stmt_ctx) @@ -131,7 +131,7 @@ impl<'a> ParserImpl<'a> { Kind::Using if { let peek_token = self.peek_token(); - peek_token.kind.is_binding_identifier() && !peek_token.is_on_new_line + peek_token.kind().is_binding_identifier() && !peek_token.is_on_new_line() } => { self.parse_using_statement() @@ -140,9 +140,9 @@ impl<'a> ParserImpl<'a> { Kind::Await if { let peek_token = self.peek_token(); - if peek_token.kind == Kind::Using && !peek_token.is_on_new_line { + if peek_token.kind() == Kind::Using && !peek_token.is_on_new_line() { let peek2_token = self.nth(2); - peek2_token.kind.is_binding_identifier() && !peek2_token.is_on_new_line + peek2_token.kind().is_binding_identifier() && !peek2_token.is_on_new_line() } else { false } @@ -572,7 +572,7 @@ impl<'a> ParserImpl<'a> { fn parse_throw_statement(&mut self) -> Statement<'a> { let span = self.start_span(); self.bump_any(); // advance `throw` - if self.cur_token().is_on_new_line { + if self.cur_token().is_on_new_line() { self.error(diagnostics::illegal_newline( "throw", self.end_span(span), diff --git a/crates/oxc_parser/src/lexer/byte_handlers.rs b/crates/oxc_parser/src/lexer/byte_handlers.rs index f87a9062c08f3..dd0198b69cde7 100644 --- a/crates/oxc_parser/src/lexer/byte_handlers.rs +++ b/crates/oxc_parser/src/lexer/byte_handlers.rs @@ -197,7 +197,7 @@ ascii_byte_handler!(SPS(lexer) { // Irregular Whitespace ascii_byte_handler!(ISP(lexer) { lexer.consume_char(); - lexer.trivia_builder.add_irregular_whitespace(lexer.token.start, lexer.offset()); + lexer.trivia_builder.add_irregular_whitespace(lexer.token.start(), lexer.offset()); Kind::Skip }); @@ -252,7 +252,7 @@ ascii_byte_handler!(HAS(lexer) { lexer.consume_char(); // HashbangComment :: // `#!` SingleLineCommentChars? - if lexer.token.start == 0 && lexer.next_ascii_byte_eq(b'!') { + if lexer.token.start() == 0 && lexer.next_ascii_byte_eq(b'!') { lexer.read_hashbang_comment() } else { lexer.private_identifier() diff --git a/crates/oxc_parser/src/lexer/comment.rs b/crates/oxc_parser/src/lexer/comment.rs index 1aea27cce39c2..400cfa52f4ee9 100644 --- a/crates/oxc_parser/src/lexer/comment.rs +++ b/crates/oxc_parser/src/lexer/comment.rs @@ -36,7 +36,7 @@ impl<'a> Lexer<'a> { if next_byte != LS_OR_PS_FIRST { // `\r` or `\n` self.trivia_builder - .add_line_comment(self.token.start, self.source.offset_of(pos), self.source.whole()); + .add_line_comment(self.token.start(), self.source.offset_of(pos), self.source.whole()); // SAFETY: Safe to consume `\r` or `\n` as both are ASCII pos = unsafe { pos.add(1) }; // We've found the end. Do not continue searching. @@ -51,7 +51,7 @@ impl<'a> Lexer<'a> { if matches!(next2, LS_BYTES_2_AND_3 | PS_BYTES_2_AND_3) { // Irregular line break self.trivia_builder - .add_line_comment(self.token.start, self.source.offset_of(pos), self.source.whole()); + .add_line_comment(self.token.start(), self.source.offset_of(pos), self.source.whole()); // Advance `pos` to after this char. // SAFETY: `0xE2` is always 1st byte of a 3-byte UTF-8 char, // so consuming 3 bytes will place `pos` on next UTF-8 char boundary. @@ -70,19 +70,19 @@ impl<'a> Lexer<'a> { } }, handle_eof: { - self.trivia_builder.add_line_comment(self.token.start, self.offset(), self.source.whole()); + self.trivia_builder.add_line_comment(self.token.start(), self.offset(), self.source.whole()); return Kind::Skip; }, }; - self.token.is_on_new_line = true; + self.token.set_is_on_new_line(true); Kind::Skip } /// Section 12.4 Multi Line Comment pub(super) fn skip_multi_line_comment(&mut self) -> Kind { // If `is_on_new_line` is already set, go directly to faster search which only looks for `*/` - if self.token.is_on_new_line { + if self.token.is_on_new_line() { return self.skip_multi_line_comment_after_line_break(self.source.position()); } @@ -119,7 +119,7 @@ impl<'a> Lexer<'a> { let next2 = unsafe { pos.add(1).read2() }; if matches!(next2, LS_BYTES_2_AND_3 | PS_BYTES_2_AND_3) { // Irregular line break - self.token.is_on_new_line = true; + self.token.set_is_on_new_line(true); // Ideally we'd go on to `skip_multi_line_comment_after_line_break` here // but can't do that easily because can't use `return` in a closure. // But irregular line breaks are rare anyway. @@ -134,7 +134,7 @@ impl<'a> Lexer<'a> { } else { // Regular line break. // No need to look for more line breaks, so switch to faster search just for `*/`. - self.token.is_on_new_line = true; + self.token.set_is_on_new_line(true); // SAFETY: Regular line breaks are ASCII, so skipping 1 byte is a UTF-8 char boundary. let after_line_break = unsafe { pos.add(1) }; return self.skip_multi_line_comment_after_line_break(after_line_break); @@ -146,7 +146,11 @@ impl<'a> Lexer<'a> { }, }; - self.trivia_builder.add_block_comment(self.token.start, self.offset(), self.source.whole()); + self.trivia_builder.add_block_comment( + self.token.start(), + self.offset(), + self.source.whole(), + ); Kind::Skip } @@ -167,7 +171,7 @@ impl<'a> Lexer<'a> { // SAFETY: `pos + index + 2` is end of `*/`, so a valid `SourcePosition` self.source.set_position(unsafe { pos.add(index + 2) }); self.trivia_builder.add_block_comment( - self.token.start, + self.token.start(), self.offset(), self.source.whole(), ); @@ -187,7 +191,7 @@ impl<'a> Lexer<'a> { } self.consume_char(); } - self.token.is_on_new_line = true; + self.token.set_is_on_new_line(true); Kind::HashbangComment } } diff --git a/crates/oxc_parser/src/lexer/jsx.rs b/crates/oxc_parser/src/lexer/jsx.rs index 2f86adb4e8c99..ae82618d2d414 100644 --- a/crates/oxc_parser/src/lexer/jsx.rs +++ b/crates/oxc_parser/src/lexer/jsx.rs @@ -54,7 +54,7 @@ impl Lexer<'_> { } pub(crate) fn next_jsx_child(&mut self) -> Token { - self.token.start = self.offset(); + self.token.set_start(self.offset()); let kind = self.read_jsx_child(); self.finish_next(kind) } diff --git a/crates/oxc_parser/src/lexer/kind.rs b/crates/oxc_parser/src/lexer/kind.rs index 66eba5f2efd8a..f178bb2469954 100644 --- a/crates/oxc_parser/src/lexer/kind.rs +++ b/crates/oxc_parser/src/lexer/kind.rs @@ -6,6 +6,7 @@ use std::fmt::{self, Display}; /// /// Exported for other oxc crates to use. You generally don't need to use this directly. #[derive(Debug, Default, Clone, Copy, Eq, PartialEq)] +#[repr(u8)] #[non_exhaustive] pub enum Kind { Undetermined, diff --git a/crates/oxc_parser/src/lexer/mod.rs b/crates/oxc_parser/src/lexer/mod.rs index 5117278e66ad8..eaae2729997f0 100644 --- a/crates/oxc_parser/src/lexer/mod.rs +++ b/crates/oxc_parser/src/lexer/mod.rs @@ -215,9 +215,9 @@ impl<'a> Lexer<'a> { } fn finish_next(&mut self, kind: Kind) -> Token { - self.token.kind = kind; - self.token.end = self.offset(); - debug_assert!(self.token.start <= self.token.end); + self.token.set_kind(kind); + self.token.set_end(self.offset()); + debug_assert!(self.token.start() <= self.token.end()); let token = self.token; self.trivia_builder.handle_token(token); self.token = Token::default(); @@ -243,7 +243,7 @@ impl<'a> Lexer<'a> { /// Get the current unterminated token range fn unterminated_range(&self) -> Span { - Span::new(self.token.start, self.offset()) + Span::new(self.token.start(), self.offset()) } /// Consume the current char if not at EOF @@ -322,7 +322,7 @@ impl<'a> Lexer<'a> { self.trivia_builder.has_no_side_effects_comment = false; loop { let offset = self.offset(); - self.token.start = offset; + self.token.set_start(offset); let Some(byte) = self.peek_byte() else { return Kind::Eof; diff --git a/crates/oxc_parser/src/lexer/punctuation.rs b/crates/oxc_parser/src/lexer/punctuation.rs index 1a2a72dc7ff35..478f1b65e033a 100644 --- a/crates/oxc_parser/src/lexer/punctuation.rs +++ b/crates/oxc_parser/src/lexer/punctuation.rs @@ -41,7 +41,7 @@ impl Lexer<'_> { match self.peek_byte() { Some(b'-') => { self.consume_char(); - if self.token.is_on_new_line + if self.token.is_on_new_line() && self.source_type.is_script() && self.next_ascii_byte_eq(b'>') { diff --git a/crates/oxc_parser/src/lexer/regex.rs b/crates/oxc_parser/src/lexer/regex.rs index 18809ecd94d2c..b750e9490a8b4 100644 --- a/crates/oxc_parser/src/lexer/regex.rs +++ b/crates/oxc_parser/src/lexer/regex.rs @@ -12,12 +12,14 @@ impl Lexer<'_> { /// Which means the parser needs to re-tokenize on `PrimaryExpression`, /// `RegularExpressionLiteral` only appear on the right hand side of `PrimaryExpression` pub(crate) fn next_regex(&mut self, kind: Kind) -> (Token, u32, RegExpFlags, bool) { - self.token.start = self.offset() - - match kind { - Kind::Slash => 1, - Kind::SlashEq => 2, - _ => unreachable!(), - }; + self.token.set_start( + self.offset() + - match kind { + Kind::Slash => 1, + Kind::SlashEq => 2, + _ => unreachable!(), + }, + ); let (pattern_end, flags, flags_error) = self.read_regex(); self.lookahead.clear(); let token = self.finish_next(Kind::RegExp); diff --git a/crates/oxc_parser/src/lexer/string.rs b/crates/oxc_parser/src/lexer/string.rs index 4078d603cb779..d6469a30b5642 100644 --- a/crates/oxc_parser/src/lexer/string.rs +++ b/crates/oxc_parser/src/lexer/string.rs @@ -166,7 +166,7 @@ macro_rules! handle_string_literal_escape { $lexer.source.next_byte_unchecked(); let next1 = $lexer.source.next_byte_unchecked(); let next2 = $lexer.source.next_byte_unchecked(); - if $lexer.token.lone_surrogates + if $lexer.token.lone_surrogates() && [next1, next2] == [LOSSY_REPLACEMENT_CHAR_BYTES[1], LOSSY_REPLACEMENT_CHAR_BYTES[2]] { let chunk = $lexer.source.str_from_pos_to_current(chunk_start); @@ -240,17 +240,17 @@ impl<'a> Lexer<'a> { if !has_escape { return; } - self.escaped_strings.insert(self.token.start, s); - self.token.escaped = true; + self.escaped_strings.insert(self.token.start(), s); + self.token.set_escaped(true); } pub(crate) fn get_string(&self, token: Token) -> &'a str { - if token.escaped { - return self.escaped_strings[&token.start]; + if token.escaped() { + return self.escaped_strings[&token.start()]; } - let raw = &self.source.whole()[token.start as usize..token.end as usize]; - match token.kind { + let raw = &self.source.whole()[token.start() as usize..token.end() as usize]; + match token.kind() { Kind::Str => { &raw[1..raw.len() - 1] // omit surrounding quotes } diff --git a/crates/oxc_parser/src/lexer/template.rs b/crates/oxc_parser/src/lexer/template.rs index 80ed89742085b..f6da2531a892b 100644 --- a/crates/oxc_parser/src/lexer/template.rs +++ b/crates/oxc_parser/src/lexer/template.rs @@ -342,7 +342,7 @@ impl<'a> Lexer<'a> { // so there must be 2 more bytes to read let next2 = unsafe { pos.add(1).read2() }; if next2 == [LOSSY_REPLACEMENT_CHAR_BYTES[1], LOSSY_REPLACEMENT_CHAR_BYTES[2]] - && self.token.lone_surrogates + && self.token.lone_surrogates() { str.push_str("\u{FFFD}fffd"); } else { @@ -387,7 +387,7 @@ impl<'a> Lexer<'a> { /// Re-tokenize the current `}` token for `TemplateSubstitutionTail` /// See Section 12, the parser needs to re-tokenize on `TemplateSubstitutionTail`, pub(crate) fn next_template_substitution_tail(&mut self) -> Token { - self.token.start = self.offset() - 1; + self.token.set_start(self.offset() - 1); let kind = self.read_template_literal(Kind::TemplateMiddle, Kind::TemplateTail); self.lookahead.clear(); self.finish_next(kind) @@ -395,8 +395,8 @@ impl<'a> Lexer<'a> { /// Save escaped template string fn save_template_string(&mut self, is_valid_escape_sequence: bool, s: &'a str) { - self.escaped_templates.insert(self.token.start, is_valid_escape_sequence.then_some(s)); - self.token.escaped = true; + self.escaped_templates.insert(self.token.start(), is_valid_escape_sequence.then_some(s)); + self.token.set_escaped(true); } pub(crate) fn get_template_string(&self, span_start: u32) -> Option<&'a str> { @@ -445,10 +445,10 @@ mod test { let mut lexer = Lexer::new(&allocator, &source_text, SourceType::default(), unique); let token = lexer.next_token(); assert_eq!( - token.kind, + token.kind(), if is_only_part { Kind::NoSubstitutionTemplate } else { Kind::TemplateHead } ); - let escaped = lexer.escaped_templates[&0]; + let escaped = lexer.escaped_templates[&token.start()]; assert_eq!(escaped, Some(expected_escaped.as_str())); } diff --git a/crates/oxc_parser/src/lexer/token.rs b/crates/oxc_parser/src/lexer/token.rs index 0ac7f60244047..221bb0bc3934c 100644 --- a/crates/oxc_parser/src/lexer/token.rs +++ b/crates/oxc_parser/src/lexer/token.rs @@ -7,16 +7,16 @@ use super::kind::Kind; #[derive(Debug, Clone, Copy, Default)] pub struct Token { /// Token Kind - pub kind: Kind, + kind: Kind, /// Start offset in source - pub start: u32, + start: u32, /// End offset in source - pub end: u32, + end: u32, /// Indicates the token is on a newline - pub is_on_new_line: bool, + is_on_new_line: bool, /// True if the identifier / string / template kinds has escaped strings. /// The escaped strings are saved in [Lexer::escaped_strings] and [Lexer::escaped_templates] by @@ -24,10 +24,10 @@ pub struct Token { /// /// [Lexer::escaped_strings]: [super::Lexer::escaped_strings] /// [Lexer::escaped_templates]: [super::Lexer::escaped_templates] - pub escaped: bool, + escaped: bool, /// True if a string contains lone surrogates. - pub lone_surrogates: bool, + lone_surrogates: bool, /// True if for numeric literal tokens that contain separator characters (`_`). /// @@ -43,32 +43,274 @@ pub struct Token { } impl Token { + #[inline] pub(super) fn new_on_new_line() -> Self { Self { is_on_new_line: true, ..Self::default() } } - pub fn span(&self) -> Span { - Span::new(self.start, self.end) + #[inline] + pub fn kind(&self) -> Kind { + self.kind + } + + #[inline] + pub fn start(&self) -> u32 { + self.start + } + + #[cfg(test)] + #[inline] + fn len(self) -> u32 { + self.end - self.start } + #[inline] + pub fn end(&self) -> u32 { + self.end + } + + #[inline] + pub fn is_on_new_line(&self) -> bool { + self.is_on_new_line + } + + #[inline] pub fn escaped(&self) -> bool { self.escaped } #[inline] - pub fn has_separator(&self) -> bool { - debug_assert!(!self.has_separator || self.kind.is_number()); + pub fn lone_surrogates(&self) -> bool { + self.lone_surrogates + } + + #[inline] + pub fn span(&self) -> Span { + Span::new(self.start, self.end) + } + + #[inline] + pub(crate) fn has_separator(&self) -> bool { self.has_separator } + #[inline] pub(crate) fn set_has_separator(&mut self) { - debug_assert!(!self.has_separator || self.kind.is_number() || self.kind == Kind::default()); self.has_separator = true; } + + // Helper to set kind, used in Default and potentially elsewhere + #[inline] + pub(crate) fn set_kind(&mut self, kind: Kind) { + self.kind = kind; + } + + #[inline] + pub(crate) fn set_start(&mut self, start: u32) { + self.start = start; + } + + #[inline] + pub(crate) fn set_is_on_new_line(&mut self, value: bool) { + self.is_on_new_line = value; + } + + #[inline] + pub(crate) fn set_escaped(&mut self, escaped: bool) { + self.escaped = escaped; + } + + #[inline] + pub(crate) fn set_lone_surrogates(&mut self, value: bool) { + self.lone_surrogates = value; + } + + #[inline] + pub(crate) fn set_end(&mut self, end: u32) { + debug_assert!(end >= self.start, "end should not be before start"); + self.end = end; + } + + #[cfg(test)] + #[inline] + fn set_len(&mut self, len: u32) { + self.end = self.start + len; + } } #[cfg(test)] -mod size_asserts { +mod test { + use super::Kind; use super::Token; + + // Test token size const _: () = assert!(std::mem::size_of::() == 16); + + // Test default token values + #[test] + fn default_token_values() { + let token = Token::default(); + assert_eq!(token.start(), 0); + assert_eq!(token.len(), 0u32); + assert_eq!(token.end(), 0); + assert_eq!(token.kind(), Kind::Eof); // Kind::default() is Eof + assert!(!token.is_on_new_line()); + assert!(!token.escaped()); + assert!(!token.lone_surrogates()); + assert!(!token.has_separator()); + } + + #[test] + fn new_on_new_line_token_values() { + let token = Token::new_on_new_line(); + assert_eq!(token.start(), 0); + assert_eq!(token.len(), 0u32); + assert_eq!(token.end(), 0); + assert_eq!(token.kind(), Kind::Eof); + assert!(token.is_on_new_line()); + assert!(!token.escaped()); + assert!(!token.lone_surrogates()); + assert!(!token.has_separator()); + } + + #[test] + fn token_creation_and_retrieval() { + let kind = Kind::Ident; + let start = 100u32; + let len = 5u32; + let is_on_new_line = true; + let escaped = false; + let lone_surrogates = true; + let has_separator = false; + + let mut token = Token::default(); + token.set_kind(kind); + token.set_start(start); + token.set_len(len); + token.set_is_on_new_line(is_on_new_line); + token.set_escaped(escaped); + token.set_lone_surrogates(lone_surrogates); + if has_separator { + // Assuming set_has_separator is not always called if false + token.set_has_separator(); + } + + assert_eq!(token.kind(), kind); + assert_eq!(token.start(), start); + assert_eq!(token.len(), len); + assert_eq!(token.end(), start + len); + assert_eq!(token.is_on_new_line(), is_on_new_line); + assert_eq!(token.escaped(), escaped); + assert_eq!(token.lone_surrogates(), lone_surrogates); + assert_eq!(token.has_separator(), has_separator); + } + + #[test] + fn token_setters() { + let mut token = Token::default(); + token.set_kind(Kind::Ident); + token.set_start(10); + token.set_len(5); + // is_on_new_line, escaped, lone_surrogates, has_separator are false by default from Token::default() + + assert_eq!(token.start(), 10); + assert!(!token.escaped()); + assert!(!token.is_on_new_line()); + assert!(!token.lone_surrogates()); + + // Test set_end + let mut token_for_set_end = Token::default(); + token_for_set_end.set_kind(Kind::Ident); + token_for_set_end.set_start(10); + token_for_set_end.set_len(5); + + assert_eq!(token_for_set_end.end(), 15); + token_for_set_end.set_end(30); + assert_eq!(token_for_set_end.start(), 10); + assert_eq!(token_for_set_end.len(), 20u32); + assert_eq!(token_for_set_end.end(), 30); + + // Test that other flags are not affected by set_start + let mut token_with_flags = Token::default(); + token_with_flags.set_kind(Kind::Str); + token_with_flags.set_start(30); + token_with_flags.set_len(3); + token_with_flags.set_is_on_new_line(true); + token_with_flags.set_escaped(true); + token_with_flags.set_lone_surrogates(true); + token_with_flags.set_has_separator(); + + token_with_flags.set_start(40); + assert_eq!(token_with_flags.start(), 40); + assert!(token_with_flags.is_on_new_line()); + assert!(token_with_flags.escaped()); + assert!(token_with_flags.lone_surrogates()); + assert!(token_with_flags.has_separator()); + + // Test that other flags are not affected by set_escaped + let mut token_with_flags2 = Token::default(); + token_with_flags2.set_kind(Kind::Str); + token_with_flags2.set_start(50); + token_with_flags2.set_len(2); + token_with_flags2.set_is_on_new_line(true); + // escaped is false by default + token_with_flags2.set_lone_surrogates(true); + token_with_flags2.set_has_separator(); + + token_with_flags2.set_escaped(true); + assert_eq!(token_with_flags2.start(), 50); + assert!(token_with_flags2.is_on_new_line()); + assert!(token_with_flags2.escaped()); + assert!(token_with_flags2.lone_surrogates()); + assert!(token_with_flags2.has_separator()); + token_with_flags2.set_escaped(false); + assert!(!token_with_flags2.escaped()); + assert!(token_with_flags2.is_on_new_line()); // Check again + assert!(token_with_flags2.lone_surrogates()); // Check again + assert!(token_with_flags2.has_separator()); // Check again + + // Test set_is_on_new_line does not affect other flags + let mut token_flags_test_newline = Token::default(); + token_flags_test_newline.set_kind(Kind::Str); + token_flags_test_newline.set_start(60); + token_flags_test_newline.set_len(2); + // is_on_new_line is false by default + token_flags_test_newline.set_escaped(true); + token_flags_test_newline.set_lone_surrogates(true); + token_flags_test_newline.set_has_separator(); + + token_flags_test_newline.set_is_on_new_line(true); + assert!(token_flags_test_newline.is_on_new_line()); + assert_eq!(token_flags_test_newline.start(), 60); + assert!(token_flags_test_newline.escaped()); + assert!(token_flags_test_newline.lone_surrogates()); + assert!(token_flags_test_newline.has_separator()); + token_flags_test_newline.set_is_on_new_line(false); + assert!(!token_flags_test_newline.is_on_new_line()); + assert!(token_flags_test_newline.escaped()); + assert!(token_flags_test_newline.lone_surrogates()); + assert!(token_flags_test_newline.has_separator()); + + // Test set_lone_surrogates does not affect other flags + let mut token_flags_test_lone_surrogates = Token::default(); + token_flags_test_lone_surrogates.set_kind(Kind::Str); + token_flags_test_lone_surrogates.set_start(70); + token_flags_test_lone_surrogates.set_len(2); + token_flags_test_lone_surrogates.set_is_on_new_line(true); + token_flags_test_lone_surrogates.set_escaped(true); + // lone_surrogates is false by default + token_flags_test_lone_surrogates.set_has_separator(); + + token_flags_test_lone_surrogates.set_lone_surrogates(true); + assert!(token_flags_test_lone_surrogates.lone_surrogates()); + assert_eq!(token_flags_test_lone_surrogates.start(), 70); + assert!(token_flags_test_lone_surrogates.is_on_new_line()); + assert!(token_flags_test_lone_surrogates.escaped()); + assert!(token_flags_test_lone_surrogates.has_separator()); + token_flags_test_lone_surrogates.set_lone_surrogates(false); + assert!(!token_flags_test_lone_surrogates.lone_surrogates()); + assert!(token_flags_test_lone_surrogates.is_on_new_line()); + assert!(token_flags_test_lone_surrogates.escaped()); + assert!(token_flags_test_lone_surrogates.has_separator()); + } } diff --git a/crates/oxc_parser/src/lexer/trivia_builder.rs b/crates/oxc_parser/src/lexer/trivia_builder.rs index a451584bd73c1..a528bd1a44e24 100644 --- a/crates/oxc_parser/src/lexer/trivia_builder.rs +++ b/crates/oxc_parser/src/lexer/trivia_builder.rs @@ -79,12 +79,12 @@ impl TriviaBuilder { pub fn handle_token(&mut self, token: Token) { let len = self.comments.len(); - self.previous_kind = token.kind; + self.previous_kind = token.kind(); if self.processed < len { // All unprocessed preceding comments are leading comments attached to this token start. for comment in &mut self.comments[self.processed..] { comment.position = CommentPosition::Leading; - comment.attached_to = token.start; + comment.attached_to = token.start(); } self.processed = len; } diff --git a/crates/oxc_parser/src/lexer/typescript.rs b/crates/oxc_parser/src/lexer/typescript.rs index d05583e6fa32f..f763a2e4461b0 100644 --- a/crates/oxc_parser/src/lexer/typescript.rs +++ b/crates/oxc_parser/src/lexer/typescript.rs @@ -8,7 +8,7 @@ impl Lexer<'_> { Kind::ShiftLeftEq => 3, _ => unreachable!(), }; - self.token.start = self.offset() - offset; + self.token.set_start(self.offset() - offset); self.source.back(offset as usize - 1); let kind = Kind::LAngle; self.lookahead.clear(); @@ -22,7 +22,7 @@ impl Lexer<'_> { Kind::ShiftRight3 => 3, _ => unreachable!(), }; - self.token.start = self.offset() - offset; + self.token.set_start(self.offset() - offset); self.source.back(offset as usize - 1); let kind = Kind::RAngle; self.lookahead.clear(); diff --git a/crates/oxc_parser/src/lexer/unicode.rs b/crates/oxc_parser/src/lexer/unicode.rs index 5fdb8ae09d547..ed98d70a72720 100644 --- a/crates/oxc_parser/src/lexer/unicode.rs +++ b/crates/oxc_parser/src/lexer/unicode.rs @@ -39,13 +39,13 @@ impl<'a> Lexer<'a> { } c if is_irregular_whitespace(c) => { self.consume_char(); - self.trivia_builder.add_irregular_whitespace(self.token.start, self.offset()); + self.trivia_builder.add_irregular_whitespace(self.token.start(), self.offset()); Kind::Skip } c if is_irregular_line_terminator(c) => { self.consume_char(); - self.token.is_on_new_line = true; - self.trivia_builder.add_irregular_whitespace(self.token.start, self.offset()); + self.token.set_is_on_new_line(true); + self.trivia_builder.add_irregular_whitespace(self.token.start(), self.offset()); Kind::Skip } _ => { @@ -135,7 +135,7 @@ impl<'a> Lexer<'a> { // For strings and templates, surrogate pairs are valid grammar, e.g. `"\uD83D\uDE00" === 😀`. match value { UnicodeEscape::CodePoint(ch) => { - if ch == '\u{FFFD}' && self.token.lone_surrogates { + if ch == '\u{FFFD}' && self.token.lone_surrogates() { // Lossy replacement character is being used as an escape marker. Escape it. text.push_str("\u{FFFD}fffd"); } else { @@ -156,8 +156,8 @@ impl<'a> Lexer<'a> { fn string_lone_surrogate(&mut self, code_point: u32, text: &mut String<'a>) { debug_assert!(code_point <= 0xFFFF); - if !self.token.lone_surrogates { - self.token.lone_surrogates = true; + if !self.token.lone_surrogates() { + self.token.set_lone_surrogates(true); // We use `\u{FFFD}` (the lossy replacement character) as a marker indicating the start // of a lone surrogate. e.g. `\u{FFFD}d800` (which will be output as `\ud800`). diff --git a/crates/oxc_parser/src/lexer/whitespace.rs b/crates/oxc_parser/src/lexer/whitespace.rs index 951c00a21bbcc..a770362a37b18 100644 --- a/crates/oxc_parser/src/lexer/whitespace.rs +++ b/crates/oxc_parser/src/lexer/whitespace.rs @@ -8,7 +8,7 @@ static NOT_REGULAR_WHITESPACE_OR_LINE_BREAK_TABLE: SafeByteMatchTable = impl Lexer<'_> { pub(super) fn line_break_handler(&mut self) -> Kind { - self.token.is_on_new_line = true; + self.token.set_is_on_new_line(true); self.trivia_builder.handle_newline(); // Indentation is common after a line break. diff --git a/crates/oxc_parser/src/modifiers.rs b/crates/oxc_parser/src/modifiers.rs index fa5fae189e49c..4d29074da5c63 100644 --- a/crates/oxc_parser/src/modifiers.rs +++ b/crates/oxc_parser/src/modifiers.rs @@ -112,7 +112,7 @@ impl TryFrom for Modifier { type Error = >::Error; fn try_from(tok: Token) -> Result { - ModifierKind::try_from(tok.kind).map(|kind| Self { span: tok.span(), kind }) + ModifierKind::try_from(tok.kind()).map(|kind| Self { span: tok.span(), kind }) } } @@ -337,7 +337,7 @@ impl<'a> ParserImpl<'a> { // Rest modifiers cannot cross line _ => { self.bump_any(); - self.can_follow_modifier() && !self.cur_token().is_on_new_line + self.can_follow_modifier() && !self.cur_token().is_on_new_line() } } } @@ -481,7 +481,7 @@ impl<'a> ParserImpl<'a> { fn next_token_is_on_same_line_and_can_follow_modifier(&mut self) -> bool { self.bump_any(); - if self.cur_token().is_on_new_line { + if self.cur_token().is_on_new_line() { return false; } self.can_follow_modifier() @@ -524,12 +524,12 @@ impl<'a> ParserImpl<'a> { fn next_token_is_class_keyword_on_same_line(&mut self) -> bool { self.bump_any(); - self.cur_kind() == Kind::Class && !self.cur_token().is_on_new_line + self.cur_kind() == Kind::Class && !self.cur_token().is_on_new_line() } fn next_token_is_function_keyword_on_same_line(&mut self) -> bool { self.bump_any(); - self.cur_kind() == Kind::Function && !self.cur_token().is_on_new_line + self.cur_kind() == Kind::Function && !self.cur_token().is_on_new_line() } fn check_for_duplicate_modifiers(&mut self, seen_flags: ModifierFlags, modifier: &Modifier) { diff --git a/crates/oxc_parser/src/ts/statement.rs b/crates/oxc_parser/src/ts/statement.rs index de9752b57be9b..375f22bb0a416 100644 --- a/crates/oxc_parser/src/ts/statement.rs +++ b/crates/oxc_parser/src/ts/statement.rs @@ -190,10 +190,10 @@ impl<'a> ParserImpl<'a> { } pub(crate) fn is_at_interface_declaration(&mut self) -> bool { - if !self.at(Kind::Interface) || self.peek_token().is_on_new_line { + if !self.at(Kind::Interface) || self.peek_token().is_on_new_line() { false } else { - self.peek_token().kind.is_binding_identifier() || self.peek_at(Kind::LCurly) + self.peek_token().kind().is_binding_identifier() || self.peek_at(Kind::LCurly) } } @@ -241,7 +241,7 @@ impl<'a> ParserImpl<'a> { pub(crate) fn is_nth_at_modifier(&mut self, n: u8, is_constructor_parameter: bool) -> bool { let nth = self.nth(n); if !(matches!( - nth.kind, + nth.kind(), Kind::Public | Kind::Protected | Kind::Private @@ -257,16 +257,16 @@ impl<'a> ParserImpl<'a> { let next = self.nth(n + 1); - if next.is_on_new_line { + if next.is_on_new_line() { false } else { let followed_by_any_member = - matches!(next.kind, Kind::PrivateIdentifier | Kind::LBrack) - || next.kind.is_literal_property_name(); - let followed_by_class_member = !is_constructor_parameter && next.kind == Kind::Star; + matches!(next.kind(), Kind::PrivateIdentifier | Kind::LBrack) + || next.kind().is_literal_property_name(); + let followed_by_class_member = !is_constructor_parameter && next.kind() == Kind::Star; // allow `...` for error recovery let followed_by_parameter = is_constructor_parameter - && matches!(next.kind, Kind::LCurly | Kind::LBrack | Kind::Dot3); + && matches!(next.kind(), Kind::LCurly | Kind::LBrack | Kind::Dot3); followed_by_any_member || followed_by_class_member || followed_by_parameter } @@ -516,11 +516,11 @@ impl<'a> ParserImpl<'a> { Kind::Interface | Kind::Type => { self.bump_any(); return self.cur_kind().is_binding_identifier() - && !self.cur_token().is_on_new_line; + && !self.cur_token().is_on_new_line(); } Kind::Module | Kind::Namespace => { self.bump_any(); - return !self.cur_token().is_on_new_line + return !self.cur_token().is_on_new_line() && (self.cur_kind().is_binding_identifier() || self.cur_kind() == Kind::Str); } @@ -533,7 +533,7 @@ impl<'a> ParserImpl<'a> { | Kind::Public | Kind::Readonly => { self.bump_any(); - if self.cur_token().is_on_new_line { + if self.cur_token().is_on_new_line() { return false; } } diff --git a/crates/oxc_parser/src/ts/types.rs b/crates/oxc_parser/src/ts/types.rs index 5e8cb665ba8cd..842b90879c654 100644 --- a/crates/oxc_parser/src/ts/types.rs +++ b/crates/oxc_parser/src/ts/types.rs @@ -17,7 +17,7 @@ impl<'a> ParserImpl<'a> { let span = self.start_span(); let ty = self.parse_union_type_or_higher(); if !self.ctx.has_disallow_conditional_types() - && !self.cur_token().is_on_new_line + && !self.cur_token().is_on_new_line() && self.eat(Kind::Extends) { let extends_type = self.context( @@ -278,7 +278,7 @@ impl<'a> ParserImpl<'a> { let span = self.start_span(); let mut ty = self.parse_non_array_type(); - while !self.cur_token().is_on_new_line { + while !self.cur_token().is_on_new_line() { match self.cur_kind() { Kind::Bang => { self.bump_any(); @@ -379,7 +379,7 @@ impl<'a> ParserImpl<'a> { let span = self.start_span(); self.bump_any(); // bump `this` let this_type = self.ast.ts_this_type(self.end_span(span)); - if self.peek_at(Kind::Is) && !self.peek_token().is_on_new_line { + if self.peek_at(Kind::Is) && !self.peek_token().is_on_new_line() { return self.parse_this_type_predicate(this_type); } TSType::TSThisType(self.alloc(this_type)) @@ -399,7 +399,7 @@ impl<'a> ParserImpl<'a> { Kind::Import => TSType::TSImportType(self.parse_ts_import_type()), Kind::Asserts => { let peek_token = self.peek_token(); - if peek_token.kind.is_identifier_name() && !peek_token.is_on_new_line { + if peek_token.kind().is_identifier_name() && !peek_token.is_on_new_line() { self.parse_asserts_type_predicate() } else { self.parse_type_reference() @@ -635,7 +635,7 @@ impl<'a> ParserImpl<'a> { } else { let entity_name = self.parse_ts_type_name(); // TODO: parseEntityName let entity_name = TSTypeQueryExprName::from(entity_name); - let type_arguments = if self.cur_token().is_on_new_line { + let type_arguments = if self.cur_token().is_on_new_line() { None } else { self.try_parse_type_arguments() @@ -788,7 +788,7 @@ impl<'a> ParserImpl<'a> { &mut self, ) -> Option>> { self.re_lex_l_angle(); - if !self.cur_token().is_on_new_line && self.re_lex_l_angle() == Kind::LAngle { + if !self.cur_token().is_on_new_line() && self.re_lex_l_angle() == Kind::LAngle { let span = self.start_span(); self.expect(Kind::LAngle); let params = self.parse_delimited_list( @@ -840,7 +840,7 @@ impl<'a> ParserImpl<'a> { Kind::LParen | Kind::NoSubstitutionTemplate | Kind::TemplateHead => true, Kind::LAngle | Kind::RAngle | Kind::Plus | Kind::Minus => false, _ => { - self.cur_token().is_on_new_line + self.cur_token().is_on_new_line() || self.is_binary_operator() || !self.is_start_of_expression() } @@ -1072,7 +1072,7 @@ impl<'a> ParserImpl<'a> { TSTypePredicateName::Identifier(self.alloc(ident_name)) }; let token = self.cur_token(); - if token.kind == Kind::Is && !token.is_on_new_line { + if token.kind() == Kind::Is && !token.is_on_new_line() { self.bump_any(); return parameter_name; } diff --git a/tasks/benchmark/benches/lexer.rs b/tasks/benchmark/benches/lexer.rs index ecf94398eaf48..6fa7d3a4a6330 100644 --- a/tasks/benchmark/benches/lexer.rs +++ b/tasks/benchmark/benches/lexer.rs @@ -47,7 +47,7 @@ fn bench_lexer(criterion: &mut Criterion) { let mut allocator = Allocator::default(); b.iter(|| { let mut lexer = Lexer::new_for_benchmarks(&allocator, source_text, source_type); - while lexer.next_token().kind != Kind::Eof {} + while lexer.next_token().kind() != Kind::Eof {} allocator.reset(); }); }); @@ -108,7 +108,7 @@ impl SourceCleaner { // Check lexer can lex it without any errors let mut lexer = Lexer::new_for_benchmarks(allocator, &self.source_text, source_type); - while lexer.next_token().kind != Kind::Eof {} + while lexer.next_token().kind() != Kind::Eof {} assert!(lexer.errors().is_empty()); }