Skip to content

refactor(parser): make Token fields private #10936

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
graphite-app merged 1 commit into from
May 11, 2025
Merged

refactor(parser): make Token fields private #10936

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 13 additions & 13 deletions crates/oxc_parser/src/cursor.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ pub struct ParserCheckpoint<'a> {
impl<'a> ParserImpl<'a> {
#[inline]
pub(crate) fn start_span(&self) -> u32 {
self.token.start
self.token.start()
}

#[inline]
Expand All @@ -39,7 +39,7 @@ impl<'a> ParserImpl<'a> {
/// Get current Kind
#[inline]
pub(crate) fn cur_kind(&self) -> Kind {
self.token.kind
self.token.kind()
}

/// Get current source text
Expand All @@ -58,7 +58,7 @@ impl<'a> ParserImpl<'a> {

/// Get current template string
pub(crate) fn cur_template_string(&self) -> Option<&'a str> {
self.lexer.get_template_string(self.token.start)
self.lexer.get_template_string(self.token.start())
}

/// Peek next token, returns EOF for final peek
Expand All @@ -70,13 +70,13 @@ impl<'a> ParserImpl<'a> {
/// Peek next kind, returns EOF for final peek
#[inline]
pub(crate) fn peek_kind(&mut self) -> Kind {
self.peek_token().kind
self.peek_token().kind()
}

/// Peek at kind
#[inline]
pub(crate) fn peek_at(&mut self, kind: Kind) -> bool {
self.peek_token().kind == kind
self.peek_token().kind() == kind
}

/// Peek nth token
Expand All @@ -91,13 +91,13 @@ impl<'a> ParserImpl<'a> {
/// Peek at nth kind
#[inline]
pub(crate) fn nth_at(&mut self, n: u8, kind: Kind) -> bool {
self.nth(n).kind == kind
self.nth(n).kind() == kind
}

/// Peek nth kind
#[inline]
pub(crate) fn nth_kind(&mut self, n: u8) -> Kind {
self.nth(n).kind
self.nth(n).kind()
}

/// Checks if the current index has token `Kind`
Expand All @@ -121,15 +121,15 @@ impl<'a> ParserImpl<'a> {
/// Checks if the current token is escaped if it is a keyword
fn advance(&mut self, kind: Kind) {
self.test_escaped_keyword(kind);
self.prev_token_end = self.token.end;
self.prev_token_end = self.token.end();
self.token = self.lexer.next_token();
}

/// Move to the next `JSXChild`
/// Checks if the current token is escaped if it is a keyword
fn advance_for_jsx_child(&mut self, kind: Kind) {
self.test_escaped_keyword(kind);
self.prev_token_end = self.token.end;
self.prev_token_end = self.token.end();
self.token = self.lexer.next_jsx_child();
}

Expand Down Expand Up @@ -182,7 +182,7 @@ impl<'a> ParserImpl<'a> {
if kind == Kind::Semicolon {
return true;
}
kind == Kind::RCurly || kind.is_eof() || self.cur_token().is_on_new_line
kind == Kind::RCurly || kind.is_eof() || self.cur_token().is_on_new_line()
}

/// # Errors
Expand Down Expand Up @@ -242,7 +242,7 @@ impl<'a> ParserImpl<'a> {
let kind = self.cur_kind();
if kind == Kind::RAngle {
self.token = self.lexer.next_right_angle();
self.token.kind
self.token.kind()
} else {
kind
}
Expand All @@ -252,7 +252,7 @@ impl<'a> ParserImpl<'a> {
let kind = self.cur_kind();
if matches!(kind, Kind::ShiftLeft | Kind::ShiftLeftEq | Kind::LtEq) {
self.token = self.lexer.re_lex_as_typescript_l_angle(kind);
self.token.kind
self.token.kind()
} else {
kind
}
Expand All @@ -262,7 +262,7 @@ impl<'a> ParserImpl<'a> {
let kind = self.cur_kind();
if matches!(kind, Kind::ShiftRight | Kind::ShiftRight3) {
self.token = self.lexer.re_lex_as_typescript_r_angle(kind);
self.token.kind
self.token.kind()
} else {
kind
}
Expand Down
14 changes: 7 additions & 7 deletions crates/oxc_parser/src/js/arrow.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,8 +65,8 @@ impl<'a> ParserImpl<'a> {

if self.at(Kind::Async) {
let second_token = self.peek_token();
let second = second_token.kind;
if second_token.is_on_new_line {
let second = second_token.kind();
if second_token.is_on_new_line() {
return Tristate::False;
}
if second != Kind::LParen && second != Kind::LAngle {
Expand Down Expand Up @@ -191,10 +191,10 @@ impl<'a> ParserImpl<'a> {
fn is_un_parenthesized_async_arrow_function_worker(&mut self) -> Tristate {
if self.at(Kind::Async) {
let first_token = self.peek_token();
let first = first_token.kind;
let first = first_token.kind();
// If the "async" is followed by "=>" token then it is not a beginning of an async arrow-function
// but instead a simple arrow-function which will be parsed inside "parseAssignmentExpressionOrHigher"
if first_token.is_on_new_line || first == Kind::Arrow {
if first_token.is_on_new_line() || first == Kind::Arrow {
return Tristate::False;
}
// Check for un-parenthesized AsyncArrowFunction
Expand Down Expand Up @@ -240,7 +240,7 @@ impl<'a> ParserImpl<'a> {

self.ctx = self.ctx.and_await(has_await);

if self.cur_token().is_on_new_line {
if self.cur_token().is_on_new_line() {
self.error(diagnostics::lineterminator_before_arrow(self.cur_token().span()));
}

Expand Down Expand Up @@ -281,7 +281,7 @@ impl<'a> ParserImpl<'a> {

self.ctx = self.ctx.and_await(has_await);

if self.cur_token().is_on_new_line {
if self.cur_token().is_on_new_line() {
self.error(diagnostics::lineterminator_before_arrow(self.cur_token().span()));
}

Expand Down Expand Up @@ -345,7 +345,7 @@ impl<'a> ParserImpl<'a> {
&mut self,
allow_return_type_in_arrow_function: bool,
) -> Option<Expression<'a>> {
let pos = self.cur_token().start;
let pos = self.cur_token().start();
if self.state.not_parenthesized_arrow.contains(&pos) {
return None;
}
Expand Down
2 changes: 1 addition & 1 deletion crates/oxc_parser/src/js/class.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -227,7 +227,7 @@ impl<'a> ParserImpl<'a> {

// async ...
if key_name.is_none() && self.at(Kind::Async) && !self.peek_at(Kind::Question) {
if !self.peek_token().is_on_new_line
if !self.peek_token().is_on_new_line()
&& (self.peek_kind().is_class_element_name_start() || self.peek_at(Kind::Star))
{
self.bump(Kind::Async);
Expand Down
2 changes: 1 addition & 1 deletion crates/oxc_parser/src/js/declaration.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ impl<'a> ParserImpl<'a> {
let mut definite = false;
if binding_kind.is_binding_identifier()
&& self.at(Kind::Bang)
&& !self.cur_token().is_on_new_line
&& !self.cur_token().is_on_new_line()
{
self.eat(Kind::Bang);
definite = true;
Expand Down
34 changes: 17 additions & 17 deletions crates/oxc_parser/src/js/expression.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -290,9 +290,9 @@ impl<'a> ParserImpl<'a> {
let span = self.start_span();
let token = self.cur_token();
let src = self.cur_src();
let value = match token.kind {
let value = match token.kind() {
Kind::Decimal | Kind::Binary | Kind::Octal | Kind::Hex => {
parse_int(src, token.kind, token.has_separator())
parse_int(src, token.kind(), token.has_separator())
}
Kind::Float | Kind::PositiveExponential | Kind::NegativeExponential => {
parse_float(src, token.has_separator())
Expand All @@ -303,7 +303,7 @@ impl<'a> ParserImpl<'a> {
self.set_fatal_error(diagnostics::invalid_number(err, token.span()));
0.0 // Dummy value
});
let base = match token.kind {
let base = match token.kind() {
Kind::Decimal => NumberBase::Decimal,
Kind::Float => NumberBase::Float,
Kind::Binary => NumberBase::Binary,
Expand Down Expand Up @@ -334,7 +334,7 @@ impl<'a> ParserImpl<'a> {
let token = self.cur_token();
let raw = self.cur_src();
let src = raw.strip_suffix('n').unwrap();
let _value = parse_big_int(src, token.kind, token.has_separator())
let _value = parse_big_int(src, token.kind(), token.has_separator())
.map_err(|err| diagnostics::invalid_number(err, token.span()));
self.bump_any();
self.ast.big_int_literal(self.end_span(span), raw, base)
Expand All @@ -346,10 +346,10 @@ impl<'a> ParserImpl<'a> {
if !self.lexer.errors.is_empty() {
return self.unexpected();
}
let pattern_start = self.cur_token().start + 1; // +1 to exclude left `/`
let pattern_start = self.cur_token().start() + 1; // +1 to exclude left `/`
let pattern_text = &self.source_text[pattern_start as usize..pattern_end as usize];
let flags_start = pattern_end + 1; // +1 to include right `/`
let flags_text = &self.source_text[flags_start as usize..self.cur_token().end as usize];
let flags_text = &self.source_text[flags_start as usize..self.cur_token().end() as usize];
let raw = self.cur_src();
self.bump_any();

Expand Down Expand Up @@ -406,7 +406,7 @@ impl<'a> ParserImpl<'a> {
}
let value = self.cur_string();
let span = self.start_span();
let lone_surrogates = self.cur_token().lone_surrogates;
let lone_surrogates = self.cur_token().lone_surrogates();
self.bump_any();
let span = self.end_span(span);
// SAFETY:
Expand Down Expand Up @@ -547,13 +547,13 @@ impl<'a> ParserImpl<'a> {
// Also replace `\r` with `\n` in `raw`.
// If contains `\r`, then `escaped` must be `true` (because `\r` needs unescaping),
// so we can skip searching for `\r` in common case where contains no escapes.
let (cooked, lone_surrogates) = if self.cur_token().escaped {
let (cooked, lone_surrogates) = if self.cur_token().escaped() {
// `cooked = None` when template literal has invalid escape sequence
let cooked = self.cur_template_string().map(Atom::from);
if cooked.is_some() && raw.contains('\r') {
raw = self.ast.atom(&raw.cow_replace("\r\n", "\n").cow_replace('\r', "\n"));
}
(cooked, self.cur_token().lone_surrogates)
(cooked, self.cur_token().lone_surrogates())
} else {
(Some(raw), false)
};
Expand Down Expand Up @@ -774,7 +774,7 @@ impl<'a> ParserImpl<'a> {
}

if !question_dot {
if self.at(Kind::Bang) && !self.cur_token().is_on_new_line && self.is_ts {
if self.at(Kind::Bang) && !self.cur_token().is_on_new_line() && self.is_ts {
self.bump_any();
lhs = self.ast.expression_ts_non_null(self.end_span(lhs_span), lhs);
continue;
Expand Down Expand Up @@ -1021,7 +1021,7 @@ impl<'a> ParserImpl<'a> {
let span = self.start_span();
let lhs = self.parse_lhs_expression_or_higher();
// ++ -- postfix update expressions
if self.cur_kind().is_update_operator() && !self.cur_token().is_on_new_line {
if self.cur_kind().is_update_operator() && !self.cur_token().is_on_new_line() {
let operator = map_update_operator(self.cur_kind());
self.bump_any();
let lhs = SimpleAssignmentTarget::cover(lhs, self);
Expand Down Expand Up @@ -1131,7 +1131,7 @@ impl<'a> ParserImpl<'a> {
}

if self.is_ts && matches!(kind, Kind::As | Kind::Satisfies) {
if self.cur_token().is_on_new_line {
if self.cur_token().is_on_new_line() {
break;
}
self.bump_any();
Expand Down Expand Up @@ -1396,7 +1396,7 @@ impl<'a> ParserImpl<'a> {
if self.at(Kind::Await) {
let peek_token = self.peek_token();
// Allow arrow expression `await => {}`
if peek_token.kind == Kind::Arrow {
if peek_token.kind() == Kind::Arrow {
return false;
}
if self.ctx.has_await() {
Expand All @@ -1405,13 +1405,13 @@ impl<'a> ParserImpl<'a> {
// The following expressions are ambiguous
// await + 0, await - 0, await ( 0 ), await [ 0 ], await / 0 /u, await ``, await of []
if matches!(
peek_token.kind,
peek_token.kind(),
Kind::Of | Kind::LParen | Kind::LBrack | Kind::Slash | Kind::RegExp
) {
return false;
}

return !peek_token.is_on_new_line && peek_token.kind.is_after_await_or_yield();
return !peek_token.is_on_new_line() && peek_token.kind().is_after_await_or_yield();
}
false
}
Expand All @@ -1420,13 +1420,13 @@ impl<'a> ParserImpl<'a> {
if self.at(Kind::Yield) {
let peek_token = self.peek_token();
// Allow arrow expression `yield => {}`
if peek_token.kind == Kind::Arrow {
if peek_token.kind() == Kind::Arrow {
return false;
}
if self.ctx.has_yield() {
return true;
}
return !peek_token.is_on_new_line && peek_token.kind.is_after_await_or_yield();
return !peek_token.is_on_new_line() && peek_token.kind().is_after_await_or_yield();
}
false
}
Expand Down
4 changes: 2 additions & 2 deletions crates/oxc_parser/src/js/function.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ impl<'a> ParserImpl<'a> {
self.at(Kind::Function)
|| self.at(Kind::Async)
&& self.peek_at(Kind::Function)
&& !self.peek_token().is_on_new_line
&& !self.peek_token().is_on_new_line()
}

pub(crate) fn parse_function_body(&mut self) -> Box<'a, FunctionBody<'a>> {
Expand Down Expand Up @@ -303,7 +303,7 @@ impl<'a> ParserImpl<'a> {
let mut delegate = false;
let mut argument = None;

if !self.cur_token().is_on_new_line {
if !self.cur_token().is_on_new_line() {
delegate = self.eat(Kind::Star);
let not_assignment_expr = matches!(
self.cur_kind(),
Expand Down
4 changes: 2 additions & 2 deletions crates/oxc_parser/src/js/module.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -172,7 +172,7 @@ impl<'a> ParserImpl<'a> {
/// [Import Attributes](https://tc39.es/proposal-import-attributes)
fn parse_import_attributes(&mut self) -> Option<WithClause<'a>> {
let attributes_keyword = match self.cur_kind() {
Kind::Assert if !self.cur_token().is_on_new_line => self.parse_identifier_name(),
Kind::Assert if !self.cur_token().is_on_new_line() => self.parse_identifier_name(),
Kind::With => self.parse_identifier_name(),
_ => {
return None;
Expand Down Expand Up @@ -403,7 +403,7 @@ impl<'a> ParserImpl<'a> {
self.parse_class_declaration(decl_span, &modifiers),
)
}
_ if self.at(Kind::Interface) && !self.peek_token().is_on_new_line && self.is_ts => {
_ if self.at(Kind::Interface) && !self.peek_token().is_on_new_line() && self.is_ts => {
let decl = self.parse_ts_interface_declaration(decl_span, &Modifiers::empty());
match decl {
Declaration::TSInterfaceDeclaration(decl) => {
Expand Down
2 changes: 1 addition & 1 deletion crates/oxc_parser/src/js/object.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ impl<'a> ParserImpl<'a> {
// AsyncGeneratorMethod
Kind::Async
if (class_element_name || peek_kind == Kind::Star)
&& !self.peek_token().is_on_new_line =>
&& !self.peek_token().is_on_new_line() =>
{
self.parse_property_definition_method()
}
Expand Down
Loading
Loading