diff --git a/apps/oxlint/test/fixtures/tokens/files/ts_angle_relex.ts b/apps/oxlint/test/fixtures/tokens/files/ts_angle_relex.ts new file mode 100644 index 0000000000000..c78aa607121f6 --- /dev/null +++ b/apps/oxlint/test/fixtures/tokens/files/ts_angle_relex.ts @@ -0,0 +1,9 @@ +// `<<` is disambiguated: speculatively tried as `<` for type args, fails, rewinds to `<<` +const a = n << 2; + +// Successful type argument parsing with `<` and `>` +const b = id(42); + +// `>` after type args is disambiguated: speculatively tried as end of type args, fails, +// rewinds to binary expression `n < (1 >> (0))` +const c = n<1>>(0); diff --git a/apps/oxlint/test/fixtures/tokens/output.snap.md b/apps/oxlint/test/fixtures/tokens/output.snap.md index 482c7e541e446..59e1d5ba61823 100644 --- a/apps/oxlint/test/fixtures/tokens/output.snap.md +++ b/apps/oxlint/test/fixtures/tokens/output.snap.md @@ -1115,6 +1115,346 @@ : ^ `---- + x tokens-plugin(tokens): Line (" `<<` is disambiguated: speculatively tried as `<` for type args, fails, rewinds to `<<`") + ,-[files/ts_angle_relex.ts:1:1] + 1 | // `<<` is disambiguated: speculatively tried as `<` for type args, fails, rewinds to `<<` + : ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + 2 | const a = n << 2; + `---- + + x tokens-plugin(tokens): Tokens and comments: + | Line loc= 1:0 - 1:90 range= 0-90 " `<<` is disambiguated: speculatively tried as `<` for type args, fails, rewinds to `<<`" + | Keyword loc= 2:0 - 2:5 range= 91-96 "const" + | Identifier loc= 2:6 - 2:7 range= 97-98 "a" + | Punctuator loc= 2:8 - 2:9 range= 99-100 "=" + | Identifier loc= 2:10 - 2:11 range= 101-102 "n" + | Punctuator loc= 2:12 - 2:14 range= 103-105 "<<" + | Numeric loc= 2:15 - 2:16 range= 106-107 "2" + | Punctuator loc= 2:16 - 2:17 range= 107-108 ";" + | Line loc= 4:0 - 4:52 range= 110-162 " Successful type argument parsing with `<` and `>`" + | Keyword loc= 5:0 - 5:5 range= 163-168 "const" + | Identifier loc= 5:6 - 5:7 range= 169-170 "b" + | Punctuator loc= 5:8 - 5:9 range= 171-172 "=" + | Identifier loc= 5:10 - 5:12 range= 173-175 "id" + | Punctuator loc= 5:12 - 5:13 range= 175-176 "<" + | Identifier loc= 5:13 - 5:19 range= 176-182 "number" + | Punctuator loc= 5:19 - 5:20 range= 182-183 ">" + | Punctuator loc= 5:20 - 5:21 range= 183-184 "(" + | Numeric loc= 5:21 - 5:23 range= 184-186 "42" + | Punctuator loc= 5:23 - 5:24 range= 186-187 ")" + | Punctuator loc= 5:24 - 5:25 range= 187-188 ";" + | Line loc= 7:0 - 7:88 range= 190-278 " `>` after type args is disambiguated: speculatively tried as end of type args, fails," + | Line loc= 8:0 - 8:48 range= 279-327 " rewinds to binary expression `n < (1 >> (0))`" + | Keyword loc= 9:0 - 9:5 range= 328-333 "const" + | Identifier loc= 9:6 - 9:7 range= 334-335 "c" + | Punctuator loc= 9:8 - 9:9 range= 336-337 "=" + | Identifier loc= 9:10 - 9:11 range= 338-339 "n" + | Punctuator loc= 9:11 - 9:12 range= 339-340 "<" + | Numeric loc= 9:12 - 9:13 range= 340-341 "1" + | Punctuator loc= 9:13 - 9:15 range= 341-343 ">>" + | Punctuator loc= 9:15 - 9:16 range= 343-344 "(" + | Numeric loc= 9:16 - 9:17 range= 344-345 "0" + | Punctuator loc= 9:17 - 9:18 range= 345-346 ")" + | Punctuator loc= 9:18 - 9:19 range= 346-347 ";" + ,-[files/ts_angle_relex.ts:1:1] + 1 | ,-> // `<<` is disambiguated: speculatively tried as `<` for type args, fails, rewinds to `<<` + 2 | | const a = n << 2; + 3 | | + 4 | | // Successful type argument parsing with `<` and `>` + 5 | | const b = id(42); + 6 | | + 7 | | // `>` after type args is disambiguated: speculatively tried as end of type args, fails, + 8 | | // rewinds to binary expression `n < (1 >> (0))` + 9 | `-> const c = n<1>>(0); + `---- + + x tokens-plugin(tokens): Tokens: + | Keyword loc= 2:0 - 2:5 range= 91-96 "const" + | Identifier loc= 2:6 - 2:7 range= 97-98 "a" + | Punctuator loc= 2:8 - 2:9 range= 99-100 "=" + | Identifier loc= 2:10 - 2:11 range= 101-102 "n" + | Punctuator loc= 2:12 - 2:14 range= 103-105 "<<" + | Numeric loc= 2:15 - 2:16 range= 106-107 "2" + | Punctuator loc= 2:16 - 2:17 range= 107-108 ";" + | Keyword loc= 5:0 - 5:5 range= 163-168 "const" + | Identifier loc= 5:6 - 5:7 range= 169-170 "b" + | Punctuator loc= 5:8 - 5:9 range= 171-172 "=" + | Identifier loc= 5:10 - 5:12 range= 173-175 "id" + | Punctuator loc= 5:12 - 5:13 range= 175-176 "<" + | Identifier loc= 5:13 - 5:19 range= 176-182 "number" + | Punctuator loc= 5:19 - 5:20 range= 182-183 ">" + | Punctuator loc= 5:20 - 5:21 range= 183-184 "(" + | Numeric loc= 5:21 - 5:23 range= 184-186 "42" + | Punctuator loc= 5:23 - 5:24 range= 186-187 ")" + | Punctuator loc= 5:24 - 5:25 range= 187-188 ";" + | Keyword loc= 9:0 - 9:5 range= 328-333 "const" + | Identifier loc= 9:6 - 9:7 range= 334-335 "c" + | Punctuator loc= 9:8 - 9:9 range= 336-337 "=" + | Identifier loc= 9:10 - 9:11 range= 338-339 "n" + | Punctuator loc= 9:11 - 9:12 range= 339-340 "<" + | Numeric loc= 9:12 - 9:13 range= 340-341 "1" + | Punctuator loc= 9:13 - 9:15 range= 341-343 ">>" + | Punctuator loc= 9:15 - 9:16 range= 343-344 "(" + | Numeric loc= 9:16 - 9:17 range= 344-345 "0" + | Punctuator loc= 9:17 - 9:18 range= 345-346 ")" + | Punctuator loc= 9:18 - 9:19 range= 346-347 ";" + ,-[files/ts_angle_relex.ts:1:1] + 1 | ,-> // `<<` is disambiguated: speculatively tried as `<` for type args, fails, rewinds to `<<` + 2 | | const a = n << 2; + 3 | | + 4 | | // Successful type argument parsing with `<` and `>` + 5 | | const b = id(42); + 6 | | + 7 | | // `>` after type args is disambiguated: speculatively tried as end of type args, fails, + 8 | | // rewinds to binary expression `n < (1 >> (0))` + 9 | `-> const c = n<1>>(0); + `---- + + x tokens-plugin(tokens): Keyword ("const") + ,-[files/ts_angle_relex.ts:2:1] + 1 | // `<<` is disambiguated: speculatively tried as `<` for type args, fails, rewinds to `<<` + 2 | const a = n << 2; + : ^^^^^ + 3 | + `---- + + x tokens-plugin(tokens): Identifier ("a") + ,-[files/ts_angle_relex.ts:2:7] + 1 | // `<<` is disambiguated: speculatively tried as `<` for type args, fails, rewinds to `<<` + 2 | const a = n << 2; + : ^ + 3 | + `---- + + x tokens-plugin(tokens): Punctuator ("=") + ,-[files/ts_angle_relex.ts:2:9] + 1 | // `<<` is disambiguated: speculatively tried as `<` for type args, fails, rewinds to `<<` + 2 | const a = n << 2; + : ^ + 3 | + `---- + + x tokens-plugin(tokens): Identifier ("n") + ,-[files/ts_angle_relex.ts:2:11] + 1 | // `<<` is disambiguated: speculatively tried as `<` for type args, fails, rewinds to `<<` + 2 | const a = n << 2; + : ^ + 3 | + `---- + + x tokens-plugin(tokens): Punctuator ("<<") + ,-[files/ts_angle_relex.ts:2:13] + 1 | // `<<` is disambiguated: speculatively tried as `<` for type args, fails, rewinds to `<<` + 2 | const a = n << 2; + : ^^ + 3 | + `---- + + x tokens-plugin(tokens): Numeric ("2") + ,-[files/ts_angle_relex.ts:2:16] + 1 | // `<<` is disambiguated: speculatively tried as `<` for type args, fails, rewinds to `<<` + 2 | const a = n << 2; + : ^ + 3 | + `---- + + x tokens-plugin(tokens): Punctuator (";") + ,-[files/ts_angle_relex.ts:2:17] + 1 | // `<<` is disambiguated: speculatively tried as `<` for type args, fails, rewinds to `<<` + 2 | const a = n << 2; + : ^ + 3 | + `---- + + x tokens-plugin(tokens): Line (" Successful type argument parsing with `<` and `>`") + ,-[files/ts_angle_relex.ts:4:1] + 3 | + 4 | // Successful type argument parsing with `<` and `>` + : ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + 5 | const b = id(42); + `---- + + x tokens-plugin(tokens): Keyword ("const") + ,-[files/ts_angle_relex.ts:5:1] + 4 | // Successful type argument parsing with `<` and `>` + 5 | const b = id(42); + : ^^^^^ + 6 | + `---- + + x tokens-plugin(tokens): Identifier ("b") + ,-[files/ts_angle_relex.ts:5:7] + 4 | // Successful type argument parsing with `<` and `>` + 5 | const b = id(42); + : ^ + 6 | + `---- + + x tokens-plugin(tokens): Punctuator ("=") + ,-[files/ts_angle_relex.ts:5:9] + 4 | // Successful type argument parsing with `<` and `>` + 5 | const b = id(42); + : ^ + 6 | + `---- + + x tokens-plugin(tokens): Identifier ("id") + ,-[files/ts_angle_relex.ts:5:11] + 4 | // Successful type argument parsing with `<` and `>` + 5 | const b = id(42); + : ^^ + 6 | + `---- + + x tokens-plugin(tokens): Punctuator ("<") + ,-[files/ts_angle_relex.ts:5:13] + 4 | // Successful type argument parsing with `<` and `>` + 5 | const b = id(42); + : ^ + 6 | + `---- + + x tokens-plugin(tokens): Identifier ("number") + ,-[files/ts_angle_relex.ts:5:14] + 4 | // Successful type argument parsing with `<` and `>` + 5 | const b = id(42); + : ^^^^^^ + 6 | + `---- + + x tokens-plugin(tokens): Punctuator (">") + ,-[files/ts_angle_relex.ts:5:20] + 4 | // Successful type argument parsing with `<` and `>` + 5 | const b = id(42); + : ^ + 6 | + `---- + + x tokens-plugin(tokens): Punctuator ("(") + ,-[files/ts_angle_relex.ts:5:21] + 4 | // Successful type argument parsing with `<` and `>` + 5 | const b = id(42); + : ^ + 6 | + `---- + + x tokens-plugin(tokens): Numeric ("42") + ,-[files/ts_angle_relex.ts:5:22] + 4 | // Successful type argument parsing with `<` and `>` + 5 | const b = id(42); + : ^^ + 6 | + `---- + + x tokens-plugin(tokens): Punctuator (")") + ,-[files/ts_angle_relex.ts:5:24] + 4 | // Successful type argument parsing with `<` and `>` + 5 | const b = id(42); + : ^ + 6 | + `---- + + x tokens-plugin(tokens): Punctuator (";") + ,-[files/ts_angle_relex.ts:5:25] + 4 | // Successful type argument parsing with `<` and `>` + 5 | const b = id(42); + : ^ + 6 | + `---- + + x tokens-plugin(tokens): Line (" `>` after type args is disambiguated: speculatively tried as end of type args, fails,") + ,-[files/ts_angle_relex.ts:7:1] + 6 | + 7 | // `>` after type args is disambiguated: speculatively tried as end of type args, fails, + : ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + 8 | // rewinds to binary expression `n < (1 >> (0))` + `---- + + x tokens-plugin(tokens): Line (" rewinds to binary expression `n < (1 >> (0))`") + ,-[files/ts_angle_relex.ts:8:1] + 7 | // `>` after type args is disambiguated: speculatively tried as end of type args, fails, + 8 | // rewinds to binary expression `n < (1 >> (0))` + : ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + 9 | const c = n<1>>(0); + `---- + + x tokens-plugin(tokens): Keyword ("const") + ,-[files/ts_angle_relex.ts:9:1] + 8 | // rewinds to binary expression `n < (1 >> (0))` + 9 | const c = n<1>>(0); + : ^^^^^ + `---- + + x tokens-plugin(tokens): Identifier ("c") + ,-[files/ts_angle_relex.ts:9:7] + 8 | // rewinds to binary expression `n < (1 >> (0))` + 9 | const c = n<1>>(0); + : ^ + `---- + + x tokens-plugin(tokens): Punctuator ("=") + ,-[files/ts_angle_relex.ts:9:9] + 8 | // rewinds to binary expression `n < (1 >> (0))` + 9 | const c = n<1>>(0); + : ^ + `---- + + x tokens-plugin(tokens): Identifier ("n") + ,-[files/ts_angle_relex.ts:9:11] + 8 | // rewinds to binary expression `n < (1 >> (0))` + 9 | const c = n<1>>(0); + : ^ + `---- + + x tokens-plugin(tokens): Punctuator ("<") + ,-[files/ts_angle_relex.ts:9:12] + 8 | // rewinds to binary expression `n < (1 >> (0))` + 9 | const c = n<1>>(0); + : ^ + `---- + + x tokens-plugin(tokens): Numeric ("1") + ,-[files/ts_angle_relex.ts:9:13] + 8 | // rewinds to binary expression `n < (1 >> (0))` + 9 | const c = n<1>>(0); + : ^ + `---- + + x tokens-plugin(tokens): Punctuator (">>") + ,-[files/ts_angle_relex.ts:9:14] + 8 | // rewinds to binary expression `n < (1 >> (0))` + 9 | const c = n<1>>(0); + : ^^ + `---- + + x tokens-plugin(tokens): Punctuator ("(") + ,-[files/ts_angle_relex.ts:9:16] + 8 | // rewinds to binary expression `n < (1 >> (0))` + 9 | const c = n<1>>(0); + : ^ + `---- + + x tokens-plugin(tokens): Numeric ("0") + ,-[files/ts_angle_relex.ts:9:17] + 8 | // rewinds to binary expression `n < (1 >> (0))` + 9 | const c = n<1>>(0); + : ^ + `---- + + x tokens-plugin(tokens): Punctuator (")") + ,-[files/ts_angle_relex.ts:9:18] + 8 | // rewinds to binary expression `n < (1 >> (0))` + 9 | const c = n<1>>(0); + : ^ + `---- + + x tokens-plugin(tokens): Punctuator (";") + ,-[files/ts_angle_relex.ts:9:19] + 8 | // rewinds to binary expression `n < (1 >> (0))` + 9 | const c = n<1>>(0); + : ^ + `---- + x tokens-plugin(tokens): Identifier ("a") ,-[files/unicode.js:1:1] 1 | a; @@ -1174,8 +1514,8 @@ : ^ `---- -Found 0 warnings and 122 errors. -Finished in Xms on 6 files with 1 rules using X threads. +Found 0 warnings and 157 errors. +Finished in Xms on 7 files with 1 rules using X threads. ``` # stderr diff --git a/crates/oxc_estree_tokens/src/lib.rs b/crates/oxc_estree_tokens/src/lib.rs index e5ef4255889e2..30a1f4c46d095 100644 --- a/crates/oxc_estree_tokens/src/lib.rs +++ b/crates/oxc_estree_tokens/src/lib.rs @@ -129,14 +129,6 @@ fn to_estree_tokens<'a>( } let span_utf16 = Span::new(start, end); - // TS estree token streams may already contain the second `<` as a - // standalone token here; skip the overlapping `<<` token. - if kind == Kind::ShiftLeft - && context.ts_type_parameter_starts.contains(&(span_utf16.start + 1)) - { - continue; - } - let is_ast_identifier = context.ast_identifier_spans.contains(&span_utf16); let is_ast_identifier = if options.exclude_legacy_keyword_identifiers { is_ast_identifier && !matches!(kind, Kind::Yield | Kind::Let | Kind::Static) @@ -183,7 +175,6 @@ pub struct EstreeTokenContext { jsx_identifier_spans: FxHashSet, non_jsx_identifier_spans: FxHashSet, jsx_text_spans: FxHashSet, - ts_type_parameter_starts: FxHashSet, jsx_expression_depth: usize, jsx_member_expression_depth: usize, jsx_computed_member_depth: usize, @@ -230,14 +221,6 @@ impl<'a> Visit<'a> for EstreeTokenContext { walk::walk_ts_import_type(self, import_type); } - fn visit_ts_type_parameter_declaration( - &mut self, - declaration: &TSTypeParameterDeclaration<'a>, - ) { - self.ts_type_parameter_starts.insert(declaration.span.start); - walk::walk_ts_type_parameter_declaration(self, declaration); - } - fn visit_identifier_name(&mut self, identifier: &IdentifierName<'a>) { self.ast_identifier_spans.insert(identifier.span); if self.member_expr_in_jsx_expression_jsx_identifiers diff --git a/crates/oxc_parser/src/js/expression.rs b/crates/oxc_parser/src/js/expression.rs index 1d61b87bf02e5..022bbbdc111e5 100644 --- a/crates/oxc_parser/src/js/expression.rs +++ b/crates/oxc_parser/src/js/expression.rs @@ -872,16 +872,22 @@ impl<'a, C: Config> ParserImpl<'a, C> { continue; } - if matches!(self.cur_kind(), Kind::LAngle | Kind::ShiftLeft) - && let Some(arguments) = + if matches!(self.cur_kind(), Kind::LAngle | Kind::ShiftLeft) { + if let Some(arguments) = self.try_parse(Self::parse_type_arguments_in_expression) - { - lhs = self.ast.expression_ts_instantiation( - self.end_span(lhs_span), - lhs, - arguments, - ); - continue; + { + lhs = self.ast.expression_ts_instantiation( + self.end_span(lhs_span), + lhs, + arguments, + ); + continue; + } + // `re_lex_as_typescript_l_angle` may have popped the original token + // (e.g. `<<`) from the collected token stream. Rewind restored the + // parser's current token, so write it back to the stream. + // This is a no-op when tokens are statically disabled (`NoTokensLexerConfig`). + self.lexer.rewrite_last_collected_token(self.token); } } @@ -1023,10 +1029,16 @@ impl<'a, C: Config> ParserImpl<'a, C> { let mut type_arguments = None; if question_dot { - if self.is_ts - && let Some(args) = self.try_parse(Self::parse_type_arguments_in_expression) - { - type_arguments = Some(args); + if self.is_ts { + if let Some(args) = self.try_parse(Self::parse_type_arguments_in_expression) { + type_arguments = Some(args); + } else { + // `re_lex_as_typescript_l_angle` may have popped the original token + // (e.g. `<<`) from the collected token stream. Rewind restored the + // parser's current token, so write it back to the stream. + // This is a no-op when tokens are statically disabled (`NoTokensLexerConfig`). + self.lexer.rewrite_last_collected_token(self.token); + } } if self.cur_kind().is_template_start_of_tagged_template() { lhs = self.parse_tagged_template(lhs_span, lhs, question_dot, type_arguments); diff --git a/crates/oxc_parser/src/lexer/mod.rs b/crates/oxc_parser/src/lexer/mod.rs index cc83d33a99f6d..85841b0beb35b 100644 --- a/crates/oxc_parser/src/lexer/mod.rs +++ b/crates/oxc_parser/src/lexer/mod.rs @@ -328,6 +328,34 @@ impl<'a, C: Config> Lexer<'a, C> { token } + /// Overwrite the last token in the collected token stream. + /// + /// Used to restore a token that was popped by `re_lex_as_typescript_l_angle` + /// when `try_parse` fails and rewinds. + #[inline] + pub(crate) fn rewrite_last_collected_token(&mut self, token: Token) { + // Make this function a no-op when tokens are statically disabled (`NoTokensLexerConfig`) + if C::TOKENS_METHOD_IS_STATIC && !self.config.tokens() { + return; + } + + // Because of the static check above, there's no need to check `self.config.tokens()` here. + // + // * If tokens are statically disabled, we already exited. + // * If tokens are statically enabled, then `self.tokens` is always non-empty. + // * If tokens are runtime disabled, then `self.tokens` is always empty. + // * If tokens are runtime enabled, then `self.tokens` is always non-empty. + // + // So checking `self.config.tokens()` too here would be redundant, + // and would be an extra branch with runtime config (`RuntimeLexerConfig`). + if let Some(last) = self.tokens.last_mut() { + *last = token; + } else { + // When tokens are enabled, this should be unreachable + debug_assert!(!self.config.tokens()); + } + } + pub(crate) fn take_tokens(&mut self) -> ArenaVec<'a, Token> { mem::replace(&mut self.tokens, ArenaVec::new_in(self.allocator)) } diff --git a/crates/oxc_parser/src/lexer/typescript.rs b/crates/oxc_parser/src/lexer/typescript.rs index 7595964f1b07c..6541f1289c1e9 100644 --- a/crates/oxc_parser/src/lexer/typescript.rs +++ b/crates/oxc_parser/src/lexer/typescript.rs @@ -3,14 +3,47 @@ use crate::config::LexerConfig as Config; use super::{Kind, Lexer, Token}; impl Lexer<'_, C> { - /// Re-tokenize '<<' or '<=' or '<<=' to '<' + /// Re-tokenize `<<`, `<=`, or `<<=` to `<`. + /// + /// Called when the parser encounters a compound token starting with `<` (e.g. `<<`) + /// in a position where it could be the start of type arguments (e.g. `foo<()>`). + /// The lexer eagerly produces the compound token, but the parser needs a single `<` + /// to speculatively try parsing type arguments. + /// + /// The compound token was pushed to the collected token stream *before* `try_parse` + /// created its checkpoint (since it was the "current" token at that point). + /// This means checkpoint's `tokens_len` includes it, and `truncate` on rewind + /// won't remove it. So we must pop it here. If the speculative parse succeeds, + /// the individual `<` and subsequent tokens replace it naturally. If it fails + /// and `try_parse` rewinds, the caller (`expression.rs`) restores the original + /// compound token via `rewrite_last_collected_token`. + /// + /// The remaining characters after the first `<` (e.g. the second `<` in `<<`) + /// will be lexed as separate tokens on subsequent `next_token` calls. pub(crate) fn re_lex_as_typescript_l_angle(&mut self, offset: u32) -> Token { self.token.set_start(self.offset() - offset); self.source.back(offset as usize - 1); + if self.config.tokens() { + let popped = self.tokens.pop(); + debug_assert!(popped.is_some()); + } self.finish_re_lex(Kind::LAngle) } - /// Re-tokenize '>>' and '>>>' to '>' + /// Re-tokenize `>>` or `>>>` to `>`. + /// + /// Called during speculative type argument parsing when the parser encounters + /// `>>` or `>>>` and needs to split off a single `>` to close the type arguments. + /// + /// Unlike `re_lex_as_typescript_l_angle`, this does NOT need to pop from the + /// collected token stream. The `>` character is initially lexed as just `RAngle` + /// (the lexer handles `>` lazily). The compound `>>` / `>>>` is only created + /// later by `re_lex_right_angle` (Replace mode), which happens *during* the + /// speculative parse (i.e. after `try_parse`'s checkpoint). This is because + /// the checkpoint is created when the parser is at `<` (the opening bracket), + /// and the `>` being replaced is the closing bracket, encountered later during + /// the parse. Since both the original push and the Replace are at a post-checkpoint + /// position, `truncate(checkpoint.tokens_len)` on rewind removes it automatically. pub(crate) fn re_lex_as_typescript_r_angle(&mut self, offset: u32) -> Token { self.token.set_start(self.offset() - offset); self.source.back(offset as usize - 1); diff --git a/oxfmtrc.jsonc b/oxfmtrc.jsonc index 8c991797d8a8a..ec7ab6bbe8604 100644 --- a/oxfmtrc.jsonc +++ b/oxfmtrc.jsonc @@ -9,6 +9,7 @@ "apps/oxlint/test/fixtures/fixes/files/**", "apps/oxlint/test/fixtures/suggestions/files/**", "apps/oxlint/test/fixtures/sourceCode_token_methods/files/**", + "apps/oxlint/test/fixtures/tokens/files/**", "**/dist/**", "**/generated/**", "**/CHANGELOG.md",