From 8d40ca33c70e0ffb699495283fd31d19320b6b88 Mon Sep 17 00:00:00 2001 From: Arsh <69170106+lilnasy@users.noreply.github.com> Date: Wed, 17 Dec 2025 07:51:55 +0530 Subject: [PATCH] feat(linter/plugins): serialize rust-parsed tokens --- apps/oxlint/src-js/generated/deserialize.js | 41 +- apps/oxlint/src-js/plugins/tokens.ts | 21 +- .../getNodeByRangeIndex/output.snap.md | 18 +- crates/oxc_ast/src/ast/js.rs | 2 + crates/oxc_ast/src/ast/mod.rs | 2 + crates/oxc_ast/src/ast/token.rs | 27 + .../oxc_ast/src/generated/assert_layouts.rs | 42 +- crates/oxc_ast/src/generated/ast_builder.rs | 6 + .../oxc_ast/src/generated/derive_clone_in.rs | 25 + .../src/generated/derive_content_eq.rs | 10 + crates/oxc_ast/src/generated/derive_dummy.rs | 1 + crates/oxc_ast/src/generated/derive_estree.rs | 13 + .../oxc_ast/src/generated/derive_get_span.rs | 8 + .../src/generated/derive_get_span_mut.rs | 8 + crates/oxc_ast/src/serialize/mod.rs | 11 +- .../oxc_ast_macros/src/generated/structs.rs | 521 +++++++++--------- crates/oxc_ast_visit/src/utf8_to_utf16/mod.rs | 1 + .../src/ast_nodes/generated/ast_nodes.rs | 5 + crates/oxc_isolated_declarations/src/lib.rs | 2 + crates/oxc_parser/src/lexer/kind.rs | 22 + crates/oxc_parser/src/lib.rs | 14 +- crates/oxc_traverse/src/generated/ancestor.rs | 16 + .../parser/src-js/generated/deserialize/js.js | 38 +- .../src-js/generated/deserialize/js_parent.js | 38 +- .../src-js/generated/deserialize/js_range.js | 39 +- .../generated/deserialize/js_range_parent.js | 39 +- .../parser/src-js/generated/deserialize/ts.js | 40 +- .../src-js/generated/deserialize/ts_parent.js | 40 +- .../src-js/generated/deserialize/ts_range.js | 41 +- .../generated/deserialize/ts_range_parent.js | 41 +- .../src-js/generated/lazy/constructors.js | 94 +++- napi/parser/src-js/generated/lazy/walk.js | 4 +- napi/parser/src/generated/assert_layouts.rs | 18 +- napi/parser/src/generated/derive_estree.rs | 1 + napi/parser/src/raw_transfer.rs | 21 +- napi/parser/src/raw_transfer_types.rs | 3 +- .../ast_tools/src/generators/raw_transfer.rs | 4 - .../ast_tools/src/generators/utf8_to_utf16.rs | 5 + tasks/ast_tools/src/main.rs | 1 + 39 files changed, 916 insertions(+), 367 deletions(-) create mode 100644 crates/oxc_ast/src/ast/token.rs diff --git a/apps/oxlint/src-js/generated/deserialize.js b/apps/oxlint/src-js/generated/deserialize.js index 5d2f24a3bcf08..9b421d5f33038 100644 --- a/apps/oxlint/src-js/generated/deserialize.js +++ b/apps/oxlint/src-js/generated/deserialize.js @@ -1,8 +1,6 @@ // Auto-generated code, DO NOT EDIT DIRECTLY! // To edit this generated file you have to edit `tasks/ast_tools/src/generators/raw_transfer.rs`. -import { tokens, initTokens } from "../plugins/tokens.js"; - let uint8, uint32, float64, @@ -53,7 +51,7 @@ function deserializeProgram(pos) { __proto__: NodeProto, type: "Program", body: null, - sourceType: deserializeModuleKind(pos + 125), + sourceType: deserializeModuleKind(pos + 149), hashbang: null, get comments() { // Check AST in buffer is still the same AST (buffers are reused) @@ -66,7 +64,9 @@ function deserializeProgram(pos) { return comments; }, get tokens() { - tokens === null && initTokens(); + if (localAstId !== astId) throw Error("Tokens are only accessible while linting the file"); + let tokens = deserializeVecToken(pos + 48); + Object.defineProperty(this, "tokens", { value: tokens }); return tokens; }, start: 0, @@ -74,9 +74,9 @@ function deserializeProgram(pos) { range: [0, end], parent: null, }); - program.hashbang = deserializeOptionHashbang(pos + 48); - let body = (program.body = deserializeVecDirective(pos + 72)); - body.push(...deserializeVecStatement(pos + 96)); + program.hashbang = deserializeOptionHashbang(pos + 72); + let body = (program.body = deserializeVecDirective(pos + 96)); + body.push(...deserializeVecStatement(pos + 120)); { let start; if (body.length > 0) { @@ -5672,6 +5672,21 @@ function deserializeComment(pos) { }; } +function deserializeToken(pos) { + let start = deserializeU32(pos), + end = deserializeU32(pos + 4); + return { + __proto__: NodeProto, + type: deserializeStr(pos + 8), + flags: deserializeOptionStr(pos + 24), + pattern: deserializeOptionStr(pos + 40), + value: sourceText.slice(start, end), + start, + end, + range: [start, end], + }; +} + function deserializeAssignmentOperator(pos) { switch (uint8[pos]) { case 0: @@ -5862,6 +5877,18 @@ function deserializeVecComment(pos) { return arr; } +function deserializeVecToken(pos) { + let arr = [], + pos32 = pos >> 2; + pos = uint32[pos32]; + let endPos = pos + uint32[pos32 + 2] * 56; + for (; pos !== endPos; ) { + arr.push(deserializeToken(pos)); + pos += 56; + } + return arr; +} + function deserializeOptionHashbang(pos) { if (uint32[(pos + 8) >> 2] === 0 && uint32[(pos + 12) >> 2] === 0) return null; return deserializeHashbang(pos); diff --git a/apps/oxlint/src-js/plugins/tokens.ts b/apps/oxlint/src-js/plugins/tokens.ts index 0c5fdecc60fbe..7cc4a66717168 100644 --- a/apps/oxlint/src-js/plugins/tokens.ts +++ b/apps/oxlint/src-js/plugins/tokens.ts @@ -2,12 +2,12 @@ * `SourceCode` methods related to tokens. */ -import { ast, initAst } from "./source_code.ts"; -import { parseTokens } from "./tokens_parse.ts"; +import { ast, initAst, sourceText } from "./source_code.ts"; import { debugAssert, debugAssertIsNonNull } from "../utils/asserts.ts"; import type { Comment, Node, NodeOrToken } from "./types.ts"; import type { Span } from "./location.ts"; +import { filePath } from "./context.ts"; /** * Options for various `SourceCode` methods e.g. `getFirstToken`. @@ -132,16 +132,19 @@ let comments: Comment[] | null = null; export let tokensAndComments: TokenOrComment[] | null = null; /** - * Initialize TS-ESLint tokens for current file. - * * Caller must ensure `filePath` and `sourceText` are initialized before calling this function. */ export function initTokens() { - // Use TypeScript parser to get tokens - tokens = parseTokens(); + debugAssertIsNonNull(filePath); + debugAssertIsNonNull(sourceText); + + if (ast === null) initAst(); + debugAssertIsNonNull(ast); + + tokens = ast.tokens; // Check `tokens` have valid ranges and are in ascending order - debugCheckValidRanges(tokens, "token"); + // debugCheckValidRanges(tokens, "token"); } /** @@ -178,7 +181,7 @@ export function initTokensAndComments() { debugAssertIsNonNull(ast); comments = ast.comments; - debugCheckValidRanges(comments, "comment"); + // debugCheckValidRanges(comments, "comment"); } // Fast paths for file with no comments, or file which is only comments @@ -289,7 +292,7 @@ function debugCheckTokensAndComments() { } } - debugCheckValidRanges(tokensAndComments, "token/comment"); + // debugCheckValidRanges(tokensAndComments, "token/comment"); } /** diff --git a/apps/oxlint/test/fixtures/getNodeByRangeIndex/output.snap.md b/apps/oxlint/test/fixtures/getNodeByRangeIndex/output.snap.md index 0526bc37557f1..9b1328e637ebd 100644 --- a/apps/oxlint/test/fixtures/getNodeByRangeIndex/output.snap.md +++ b/apps/oxlint/test/fixtures/getNodeByRangeIndex/output.snap.md @@ -186,6 +186,14 @@ 6 | `---- + x getNode-plugin(getNode): type: TemplateElement + ,-[files/index.ts:5:11] + 4 | + 5 | `___${123}___`; + : ^ + 6 | + `---- + x getNode-plugin(getNode): type: TemplateElement ,-[files/index.ts:5:14] 4 | @@ -314,6 +322,14 @@ 8 | `---- + x getNode-plugin(getNode): type: TemplateElement + ,-[files/index.ts:7:20] + 6 | + 7 | type T = `___${123}___`; + : ^ + 8 | + `---- + x getNode-plugin(getNode): type: TemplateElement ,-[files/index.ts:7:23] 6 | @@ -355,7 +371,7 @@ 9 | // Comment `---- -Found 0 warnings and 45 errors. +Found 0 warnings and 47 errors. Finished in Xms on 1 file using X threads. ``` diff --git a/crates/oxc_ast/src/ast/js.rs b/crates/oxc_ast/src/ast/js.rs index ef33f963a7480..c87816743a2c9 100644 --- a/crates/oxc_ast/src/ast/js.rs +++ b/crates/oxc_ast/src/ast/js.rs @@ -57,6 +57,8 @@ pub struct Program<'a> { #[content_eq(skip)] #[estree(skip)] pub comments: Vec<'a, Comment>, + #[estree(skip)] + pub tokens: Vec<'a, Token<'a>>, pub hashbang: Option>, #[estree(prepend_to = body)] pub directives: Vec<'a, Directive<'a>>, diff --git a/crates/oxc_ast/src/ast/mod.rs b/crates/oxc_ast/src/ast/mod.rs index d9a022f368068..846eeb789a73f 100644 --- a/crates/oxc_ast/src/ast/mod.rs +++ b/crates/oxc_ast/src/ast/mod.rs @@ -187,12 +187,14 @@ pub(crate) mod js; pub(crate) mod jsx; pub(crate) mod literal; pub(crate) mod macros; +pub(crate) mod token; pub(crate) mod ts; pub use comment::*; pub use js::*; pub use jsx::*; pub use literal::*; +pub use token::*; pub use ts::*; use macros::inherit_variants; diff --git a/crates/oxc_ast/src/ast/token.rs b/crates/oxc_ast/src/ast/token.rs new file mode 100644 index 0000000000000..547a71db22cb4 --- /dev/null +++ b/crates/oxc_ast/src/ast/token.rs @@ -0,0 +1,27 @@ +use oxc_allocator::CloneIn; +use oxc_ast_macros::{ast, ast_meta}; +use oxc_estree::ESTree; +use oxc_span::{Atom, ContentEq, GetSpan, GetSpanMut, Span}; + +#[ast] +#[generate_derive(CloneIn, ContentEq, ESTree, GetSpan, GetSpanMut)] +#[estree(add_fields(value = TokenValue), no_type, no_ts_def, no_parent)] +#[derive(Debug)] +/// Represents a token in the source code. +pub struct Token<'a> { + /// Span. + #[span] + pub span: Span, + /// Type. + pub r#type: Atom<'a>, + /// Flags. + pub flags: Option>, + /// Pattern. + pub pattern: Option>, +} + +/// Custom deserializer for `value` field of `Token`. +#[ast_meta] +#[generate_derive(CloneIn, ContentEq, ESTree)] +#[estree(ts_type = "string", raw_deser = "SOURCE_TEXT.slice(THIS.start, THIS.end)")] +pub struct TokenValue<'a, 'b>(pub &'b Token<'a>); diff --git a/crates/oxc_ast/src/generated/assert_layouts.rs b/crates/oxc_ast/src/generated/assert_layouts.rs index e60ababfb5168..5c7019be5f17c 100644 --- a/crates/oxc_ast/src/generated/assert_layouts.rs +++ b/crates/oxc_ast/src/generated/assert_layouts.rs @@ -10,16 +10,17 @@ use crate::ast::*; #[cfg(target_pointer_width = "64")] const _: () = { // Padding: 1 bytes - assert!(size_of::() == 128); + assert!(size_of::() == 152); assert!(align_of::() == 8); assert!(offset_of!(Program, span) == 0); - assert!(offset_of!(Program, source_type) == 124); + assert!(offset_of!(Program, source_type) == 148); assert!(offset_of!(Program, source_text) == 8); assert!(offset_of!(Program, comments) == 24); - assert!(offset_of!(Program, hashbang) == 48); - assert!(offset_of!(Program, directives) == 72); - assert!(offset_of!(Program, body) == 96); - assert!(offset_of!(Program, scope_id) == 120); + assert!(offset_of!(Program, tokens) == 48); + assert!(offset_of!(Program, hashbang) == 72); + assert!(offset_of!(Program, directives) == 96); + assert!(offset_of!(Program, body) == 120); + assert!(offset_of!(Program, scope_id) == 144); assert!(size_of::() == 16); assert!(align_of::() == 8); @@ -1621,21 +1622,30 @@ const _: () = { assert!(offset_of!(Comment, position) == 13); assert!(offset_of!(Comment, newlines) == 14); assert!(offset_of!(Comment, content) == 15); + + // Padding: 0 bytes + assert!(size_of::() == 56); + assert!(align_of::() == 8); + assert!(offset_of!(Token, span) == 0); + assert!(offset_of!(Token, r#type) == 8); + assert!(offset_of!(Token, flags) == 24); + assert!(offset_of!(Token, pattern) == 40); }; #[cfg(target_pointer_width = "32")] const _: () = if cfg!(target_family = "wasm") || align_of::() == 8 { // Padding: 1 bytes - assert!(size_of::() == 88); + assert!(size_of::() == 104); assert!(align_of::() == 4); assert!(offset_of!(Program, span) == 0); - assert!(offset_of!(Program, source_type) == 84); + assert!(offset_of!(Program, source_type) == 100); assert!(offset_of!(Program, source_text) == 8); assert!(offset_of!(Program, comments) == 16); - assert!(offset_of!(Program, hashbang) == 32); - assert!(offset_of!(Program, directives) == 48); - assert!(offset_of!(Program, body) == 64); - assert!(offset_of!(Program, scope_id) == 80); + assert!(offset_of!(Program, tokens) == 32); + assert!(offset_of!(Program, hashbang) == 48); + assert!(offset_of!(Program, directives) == 64); + assert!(offset_of!(Program, body) == 80); + assert!(offset_of!(Program, scope_id) == 96); assert!(size_of::() == 8); assert!(align_of::() == 4); @@ -3237,6 +3247,14 @@ const _: () = if cfg!(target_family = "wasm") || align_of::() == 8 { assert!(offset_of!(Comment, position) == 13); assert!(offset_of!(Comment, newlines) == 14); assert!(offset_of!(Comment, content) == 15); + + // Padding: 0 bytes + assert!(size_of::() == 32); + assert!(align_of::() == 4); + assert!(offset_of!(Token, span) == 0); + assert!(offset_of!(Token, r#type) == 8); + assert!(offset_of!(Token, flags) == 16); + assert!(offset_of!(Token, pattern) == 24); }; #[cfg(not(any(target_pointer_width = "64", target_pointer_width = "32")))] diff --git a/crates/oxc_ast/src/generated/ast_builder.rs b/crates/oxc_ast/src/generated/ast_builder.rs index 3a4a1c090aea8..d27490dbd45ab 100644 --- a/crates/oxc_ast/src/generated/ast_builder.rs +++ b/crates/oxc_ast/src/generated/ast_builder.rs @@ -27,6 +27,7 @@ impl<'a> AstBuilder<'a> { /// * `source_type` /// * `source_text` /// * `comments`: Sorted comments + /// * `tokens` /// * `hashbang` /// * `directives` /// * `body` @@ -37,6 +38,7 @@ impl<'a> AstBuilder<'a> { source_type: SourceType, source_text: &'a str, comments: Vec<'a, Comment>, + tokens: Vec<'a, Token<'a>>, hashbang: Option>, directives: Vec<'a, Directive<'a>>, body: Vec<'a, Statement<'a>>, @@ -46,6 +48,7 @@ impl<'a> AstBuilder<'a> { source_type, source_text, comments, + tokens, hashbang, directives, body, @@ -60,6 +63,7 @@ impl<'a> AstBuilder<'a> { /// * `source_type` /// * `source_text` /// * `comments`: Sorted comments + /// * `tokens` /// * `hashbang` /// * `directives` /// * `body` @@ -71,6 +75,7 @@ impl<'a> AstBuilder<'a> { source_type: SourceType, source_text: &'a str, comments: Vec<'a, Comment>, + tokens: Vec<'a, Token<'a>>, hashbang: Option>, directives: Vec<'a, Directive<'a>>, body: Vec<'a, Statement<'a>>, @@ -81,6 +86,7 @@ impl<'a> AstBuilder<'a> { source_type, source_text, comments, + tokens, hashbang, directives, body, diff --git a/crates/oxc_ast/src/generated/derive_clone_in.rs b/crates/oxc_ast/src/generated/derive_clone_in.rs index d878ec54913e8..49b12d6bb921e 100644 --- a/crates/oxc_ast/src/generated/derive_clone_in.rs +++ b/crates/oxc_ast/src/generated/derive_clone_in.rs @@ -9,6 +9,7 @@ use crate::ast::comment::*; use crate::ast::js::*; use crate::ast::jsx::*; use crate::ast::literal::*; +use crate::ast::token::*; use crate::ast::ts::*; impl<'new_alloc> CloneIn<'new_alloc> for Program<'_> { @@ -20,6 +21,7 @@ impl<'new_alloc> CloneIn<'new_alloc> for Program<'_> { source_type: CloneIn::clone_in(&self.source_type, allocator), source_text: CloneIn::clone_in(&self.source_text, allocator), comments: CloneIn::clone_in(&self.comments, allocator), + tokens: CloneIn::clone_in(&self.tokens, allocator), hashbang: CloneIn::clone_in(&self.hashbang, allocator), directives: CloneIn::clone_in(&self.directives, allocator), body: CloneIn::clone_in(&self.body, allocator), @@ -33,6 +35,7 @@ impl<'new_alloc> CloneIn<'new_alloc> for Program<'_> { source_type: CloneIn::clone_in_with_semantic_ids(&self.source_type, allocator), source_text: CloneIn::clone_in_with_semantic_ids(&self.source_text, allocator), comments: CloneIn::clone_in_with_semantic_ids(&self.comments, allocator), + tokens: CloneIn::clone_in_with_semantic_ids(&self.tokens, allocator), hashbang: CloneIn::clone_in_with_semantic_ids(&self.hashbang, allocator), directives: CloneIn::clone_in_with_semantic_ids(&self.directives, allocator), body: CloneIn::clone_in_with_semantic_ids(&self.body, allocator), @@ -8012,3 +8015,25 @@ impl<'new_alloc> CloneIn<'new_alloc> for Comment { } } } + +impl<'new_alloc> CloneIn<'new_alloc> for Token<'_> { + type Cloned = Token<'new_alloc>; + + fn clone_in(&self, allocator: &'new_alloc Allocator) -> Self::Cloned { + Token { + span: CloneIn::clone_in(&self.span, allocator), + r#type: CloneIn::clone_in(&self.r#type, allocator), + flags: CloneIn::clone_in(&self.flags, allocator), + pattern: CloneIn::clone_in(&self.pattern, allocator), + } + } + + fn clone_in_with_semantic_ids(&self, allocator: &'new_alloc Allocator) -> Self::Cloned { + Token { + span: CloneIn::clone_in_with_semantic_ids(&self.span, allocator), + r#type: CloneIn::clone_in_with_semantic_ids(&self.r#type, allocator), + flags: CloneIn::clone_in_with_semantic_ids(&self.flags, allocator), + pattern: CloneIn::clone_in_with_semantic_ids(&self.pattern, allocator), + } + } +} diff --git a/crates/oxc_ast/src/generated/derive_content_eq.rs b/crates/oxc_ast/src/generated/derive_content_eq.rs index 3458a4817ab81..a1eadc4eb612c 100644 --- a/crates/oxc_ast/src/generated/derive_content_eq.rs +++ b/crates/oxc_ast/src/generated/derive_content_eq.rs @@ -9,11 +9,13 @@ use crate::ast::comment::*; use crate::ast::js::*; use crate::ast::jsx::*; use crate::ast::literal::*; +use crate::ast::token::*; use crate::ast::ts::*; impl ContentEq for Program<'_> { fn content_eq(&self, other: &Self) -> bool { ContentEq::content_eq(&self.source_type, &other.source_type) + && ContentEq::content_eq(&self.tokens, &other.tokens) && ContentEq::content_eq(&self.hashbang, &other.hashbang) && ContentEq::content_eq(&self.directives, &other.directives) && ContentEq::content_eq(&self.body, &other.body) @@ -2525,3 +2527,11 @@ impl ContentEq for Comment { && ContentEq::content_eq(&self.content, &other.content) } } + +impl ContentEq for Token<'_> { + fn content_eq(&self, other: &Self) -> bool { + ContentEq::content_eq(&self.r#type, &other.r#type) + && ContentEq::content_eq(&self.flags, &other.flags) + && ContentEq::content_eq(&self.pattern, &other.pattern) + } +} diff --git a/crates/oxc_ast/src/generated/derive_dummy.rs b/crates/oxc_ast/src/generated/derive_dummy.rs index 08e9d8ba7854a..85f9e7130c7a7 100644 --- a/crates/oxc_ast/src/generated/derive_dummy.rs +++ b/crates/oxc_ast/src/generated/derive_dummy.rs @@ -20,6 +20,7 @@ impl<'a> Dummy<'a> for Program<'a> { source_type: Dummy::dummy(allocator), source_text: Dummy::dummy(allocator), comments: Dummy::dummy(allocator), + tokens: Dummy::dummy(allocator), hashbang: Dummy::dummy(allocator), directives: Dummy::dummy(allocator), body: Dummy::dummy(allocator), diff --git a/crates/oxc_ast/src/generated/derive_estree.rs b/crates/oxc_ast/src/generated/derive_estree.rs index 1d0cadc00a1be..3f2b6dcf5b4cd 100644 --- a/crates/oxc_ast/src/generated/derive_estree.rs +++ b/crates/oxc_ast/src/generated/derive_estree.rs @@ -11,6 +11,7 @@ use crate::ast::comment::*; use crate::ast::js::*; use crate::ast::jsx::*; use crate::ast::literal::*; +use crate::ast::token::*; use crate::ast::ts::*; impl ESTree for Program<'_> { @@ -3248,3 +3249,15 @@ impl ESTree for Comment { state.end(); } } + +impl ESTree for Token<'_> { + fn serialize(&self, serializer: S) { + let mut state = serializer.serialize_struct(); + state.serialize_field("type", &self.r#type); + state.serialize_field("flags", &self.flags); + state.serialize_field("pattern", &self.pattern); + state.serialize_field("value", &crate::ast::token::TokenValue(self)); + state.serialize_span(self.span); + state.end(); + } +} diff --git a/crates/oxc_ast/src/generated/derive_get_span.rs b/crates/oxc_ast/src/generated/derive_get_span.rs index e6648a3557193..5e6ba2de2d7c9 100644 --- a/crates/oxc_ast/src/generated/derive_get_span.rs +++ b/crates/oxc_ast/src/generated/derive_get_span.rs @@ -8,6 +8,7 @@ use oxc_span::{GetSpan, Span}; use crate::ast::js::*; use crate::ast::jsx::*; use crate::ast::literal::*; +use crate::ast::token::*; use crate::ast::ts::*; impl GetSpan for Program<'_> { @@ -2180,3 +2181,10 @@ impl GetSpan for JSDocUnknownType { self.span } } + +impl GetSpan for Token<'_> { + #[inline] + fn span(&self) -> Span { + GetSpan::span(&self.span) + } +} diff --git a/crates/oxc_ast/src/generated/derive_get_span_mut.rs b/crates/oxc_ast/src/generated/derive_get_span_mut.rs index 349258f985863..9d5f01f809f73 100644 --- a/crates/oxc_ast/src/generated/derive_get_span_mut.rs +++ b/crates/oxc_ast/src/generated/derive_get_span_mut.rs @@ -8,6 +8,7 @@ use oxc_span::{GetSpanMut, Span}; use crate::ast::js::*; use crate::ast::jsx::*; use crate::ast::literal::*; +use crate::ast::token::*; use crate::ast::ts::*; impl GetSpanMut for Program<'_> { @@ -2180,3 +2181,10 @@ impl GetSpanMut for JSDocUnknownType { &mut self.span } } + +impl GetSpanMut for Token<'_> { + #[inline] + fn span_mut(&mut self) -> &mut Span { + GetSpanMut::span_mut(&mut self.span) + } +} diff --git a/crates/oxc_ast/src/serialize/mod.rs b/crates/oxc_ast/src/serialize/mod.rs index a2b1970cd1134..ef5a91f034e84 100644 --- a/crates/oxc_ast/src/serialize/mod.rs +++ b/crates/oxc_ast/src/serialize/mod.rs @@ -144,7 +144,9 @@ impl Program<'_> { /* END_IF */ /* IF LINTER */ get tokens() { - if (tokens === null) initTokens(); + if (localAstId !== astId) throw new Error('Tokens are only accessible while linting the file'); + const tokens = DESER[Vec](POS_OFFSET.tokens); + Object.defineProperty(this, 'tokens', { value: tokens }); return tokens; }, /* END_IF */ @@ -263,3 +265,10 @@ impl ESTree for CommentValue<'_> { unimplemented!(); } } + +impl ESTree for crate::ast::token::TokenValue<'_, '_> { + #[expect(clippy::unimplemented)] + fn serialize(&self, _serializer: S) { + unimplemented!(); + } +} diff --git a/crates/oxc_ast_macros/src/generated/structs.rs b/crates/oxc_ast_macros/src/generated/structs.rs index 7985d92339ef9..1bd2e36f4a4fa 100644 --- a/crates/oxc_ast_macros/src/generated/structs.rs +++ b/crates/oxc_ast_macros/src/generated/structs.rs @@ -9,302 +9,303 @@ pub static STRUCTS: phf::Map<&'static str, StructDetails> = ::phf::Map { key: 16287231350648472473, disps: &[ (0, 0), - (0, 89), - (0, 14), - (0, 16), - (0, 49), - (3, 197), - (7, 117), - (0, 60), - (16, 193), - (0, 28), - (0, 81), - (0, 26), + (0, 70), + (0, 48), (0, 2), - (0, 3), + (1, 112), + (8, 191), + (0, 221), + (1, 24), + (1, 182), + (1, 65), + (0, 127), + (0, 5), + (0, 55), + (0, 145), + (0, 25), + (0, 2), + (0, 45), + (5, 127), + (5, 226), + (0, 83), + (7, 32), + (0, 0), + (1, 32), (1, 74), + (0, 10), + (0, 52), + (0, 36), + (0, 2), + (2, 92), + (5, 138), + (3, 118), + (0, 9), + (5, 208), (0, 1), - (0, 14), - (10, 193), - (6, 35), - (0, 227), - (4, 109), - (0, 8), - (0, 116), - (0, 142), - (0, 0), - (0, 65), - (1, 35), - (0, 21), - (0, 141), - (10, 18), - (0, 27), - (0, 58), - (15, 157), + (4, 142), + (2, 136), + (0, 10), + (14, 98), (0, 0), - (20, 30), - (6, 20), - (0, 30), - (33, 128), - (0, 0), - (3, 83), - (1, 165), - (0, 9), - (20, 139), - (1, 27), - (0, 37), - (22, 39), - (0, 4), + (36, 0), + (0, 14), + (0, 26), + (47, 221), + (0, 42), + (2, 34), + (38, 81), + (0, 3), ], entries: &[ - ("Elision", StructDetails { field_order: None }), - ("TSOptionalType", StructDetails { field_order: None }), - ("WithClause", StructDetails { field_order: Some(&[0, 2, 1]) }), - ("NewExpression", StructDetails { field_order: None }), - ("NumericLiteral", StructDetails { field_order: None }), - ("TSInterfaceDeclaration", StructDetails { field_order: Some(&[0, 1, 2, 3, 4, 6, 5]) }), - ("UnicodePropertyEscape", StructDetails { field_order: Some(&[0, 3, 4, 1, 2]) }), - ("TSBooleanKeyword", StructDetails { field_order: None }), - ("ObjectAssignmentTarget", StructDetails { field_order: None }), - ("SwitchCase", StructDetails { field_order: None }), - ("AssignmentTargetPropertyIdentifier", StructDetails { field_order: None }), - ("CatchClause", StructDetails { field_order: None }), - ("RegExpFlags", StructDetails { field_order: None }), - ("Hashbang", StructDetails { field_order: None }), - ("BlockStatement", StructDetails { field_order: None }), - ("StringLiteral", StructDetails { field_order: None }), - ("TSNumberKeyword", StructDetails { field_order: None }), - ("ChainExpression", StructDetails { field_order: None }), - ("TSTypeParameterInstantiation", StructDetails { field_order: None }), - ("ScopeId", StructDetails { field_order: None }), - ("ImportDeclaration", StructDetails { field_order: Some(&[0, 1, 2, 4, 3, 5]) }), - ("SymbolId", StructDetails { field_order: None }), - ("RegExpPattern", StructDetails { field_order: None }), - ("TSModuleBlock", StructDetails { field_order: None }), - ("TSObjectKeyword", StructDetails { field_order: None }), - ("JSXNamespacedName", StructDetails { field_order: None }), - ("CapturingGroup", StructDetails { field_order: None }), - ("BindingRestElement", StructDetails { field_order: None }), - ("TSTypeQuery", StructDetails { field_order: None }), - ("SpreadElement", StructDetails { field_order: None }), - ("RawTransferMetadata2", StructDetails { field_order: Some(&[1, 2, 0]) }), - ("TSAsExpression", StructDetails { field_order: None }), - ("ReturnStatement", StructDetails { field_order: None }), - ("ArrayAssignmentTarget", StructDetails { field_order: None }), - ("BindingPattern", StructDetails { field_order: None }), - ("V8IntrinsicExpression", StructDetails { field_order: None }), - ("TSUndefinedKeyword", StructDetails { field_order: None }), - ("CatchParameter", StructDetails { field_order: None }), - ("RawTransferData", StructDetails { field_order: None }), - ("RawTransferMetadata", StructDetails { field_order: Some(&[1, 2, 0]) }), - ("ObjectProperty", StructDetails { field_order: Some(&[0, 3, 1, 2, 4, 5, 6]) }), - ("ThisExpression", StructDetails { field_order: None }), - ("TSModuleDeclaration", StructDetails { field_order: Some(&[0, 1, 2, 4, 5, 3]) }), - ("ImportNamespaceSpecifier", StructDetails { field_order: None }), - ("ExportEntry", StructDetails { field_order: Some(&[1, 0, 2, 3, 4, 5, 6]) }), - ("DoWhileStatement", StructDetails { field_order: None }), - ("FunctionBody", StructDetails { field_order: None }), - ("Dot", StructDetails { field_order: None }), - ("UnaryExpression", StructDetails { field_order: Some(&[0, 2, 1]) }), - ("TSLiteralType", StructDetails { field_order: None }), - ("ObjectExpression", StructDetails { field_order: None }), - ("JSDocNullableType", StructDetails { field_order: None }), - ("TSBigIntKeyword", StructDetails { field_order: None }), - ("JSXSpreadChild", StructDetails { field_order: None }), - ("AssignmentTargetRest", StructDetails { field_order: None }), - ("IfStatement", StructDetails { field_order: None }), - ("TSTypeAnnotation", StructDetails { field_order: None }), - ("LookAroundAssertion", StructDetails { field_order: Some(&[0, 2, 1]) }), - ("EcmaScriptModule", StructDetails { field_order: Some(&[4, 0, 1, 2, 3]) }), - ("RegExp", StructDetails { field_order: None }), - ( - "ArrowFunctionExpression", - StructDetails { field_order: Some(&[0, 6, 7, 1, 2, 3, 4, 5, 8, 9]) }, - ), - ("Span", StructDetails { field_order: None }), - ("BigIntLiteral", StructDetails { field_order: None }), ("AssignmentExpression", StructDetails { field_order: Some(&[0, 3, 1, 2]) }), - ("ConditionalExpression", StructDetails { field_order: None }), + ("TSClassImplements", StructDetails { field_order: None }), + ("PrivateInExpression", StructDetails { field_order: None }), + ("IdentifierReference", StructDetails { field_order: None }), + ("SwitchStatement", StructDetails { field_order: None }), + ("TSParenthesizedType", StructDetails { field_order: None }), + ("LabeledStatement", StructDetails { field_order: None }), + ("ExportAllDeclaration", StructDetails { field_order: None }), + ("TSTypeAnnotation", StructDetails { field_order: None }), + ("Hashbang", StructDetails { field_order: None }), + ("TSTypeAliasDeclaration", StructDetails { field_order: Some(&[0, 1, 2, 3, 5, 4]) }), + ("StaticBlock", StructDetails { field_order: None }), ( "PropertyDefinition", StructDetails { field_order: Some(&[0, 5, 1, 2, 3, 4, 6, 7, 8, 9, 10, 11, 12, 13]) }, ), - ("Comment", StructDetails { field_order: None }), + ("TryStatement", StructDetails { field_order: None }), + ("ForInStatement", StructDetails { field_order: None }), + ("TemplateElementValue", StructDetails { field_order: None }), + ("BindingRestElement", StructDetails { field_order: None }), + ("ExportDefaultDeclaration", StructDetails { field_order: None }), + ("TSTemplateLiteralType", StructDetails { field_order: None }), + ("TSTupleType", StructDetails { field_order: None }), + ("TSIntersectionType", StructDetails { field_order: None }), + ("AssignmentPattern", StructDetails { field_order: None }), + ("JSXMemberExpression", StructDetails { field_order: None }), + ("TSNonNullExpression", StructDetails { field_order: None }), + ("ExpressionStatement", StructDetails { field_order: None }), + ("TSNamedTupleMember", StructDetails { field_order: None }), + ("CapturingGroup", StructDetails { field_order: None }), + ("TaggedTemplateExpression", StructDetails { field_order: None }), + ("TSBooleanKeyword", StructDetails { field_order: None }), ("DebuggerStatement", StructDetails { field_order: None }), - ("StaticMemberExpression", StructDetails { field_order: None }), - ("CharacterClassEscape", StructDetails { field_order: None }), - ("TSUnknownKeyword", StructDetails { field_order: None }), + ("AssignmentTargetRest", StructDetails { field_order: None }), + ("ObjectPattern", StructDetails { field_order: None }), + ("ImportSpecifier", StructDetails { field_order: None }), + ("Error", StructDetails { field_order: Some(&[4, 0, 1, 2, 3]) }), + ("Quantifier", StructDetails { field_order: Some(&[0, 1, 2, 4, 3]) }), + ("DynamicImport", StructDetails { field_order: None }), + ("JSXExpressionContainer", StructDetails { field_order: None }), + ("TSNamespaceExportDeclaration", StructDetails { field_order: None }), + ("ArrayPattern", StructDetails { field_order: None }), + ("TSUndefinedKeyword", StructDetails { field_order: None }), + ("FormalParameter", StructDetails { field_order: None }), + ("TSInterfaceBody", StructDetails { field_order: None }), + ("ConditionalExpression", StructDetails { field_order: None }), + ("JSXNamespacedName", StructDetails { field_order: None }), + ("EmptyStatement", StructDetails { field_order: None }), + ("SequenceExpression", StructDetails { field_order: None }), + ("Comment", StructDetails { field_order: None }), + ( + "Function", + StructDetails { field_order: Some(&[0, 8, 1, 9, 10, 11, 2, 3, 4, 5, 6, 7, 12, 13]) }, + ), + ("TSSymbolKeyword", StructDetails { field_order: None }), ("JSDocUnknownType", StructDetails { field_order: None }), + ("ObjectAssignmentTarget", StructDetails { field_order: None }), + ("TSIndexSignatureName", StructDetails { field_order: None }), + ("Modifiers", StructDetails { field_order: None }), + ("TSRestType", StructDetails { field_order: None }), ("ErrorLabel", StructDetails { field_order: Some(&[1, 0]) }), - ("TSConstructSignatureDeclaration", StructDetails { field_order: None }), + ("ImportExpression", StructDetails { field_order: None }), + ("CatchClause", StructDetails { field_order: None }), + ("TSImportType", StructDetails { field_order: None }), + ("TSTypeOperator", StructDetails { field_order: Some(&[0, 2, 1]) }), + ("NamedReference", StructDetails { field_order: None }), ( "MethodDefinition", StructDetails { field_order: Some(&[0, 4, 1, 2, 3, 5, 6, 7, 8, 9, 10]) }, ), + ("TSInterfaceDeclaration", StructDetails { field_order: Some(&[0, 1, 2, 3, 4, 6, 5]) }), + ("NumericLiteral", StructDetails { field_order: None }), + ("PrivateIdentifier", StructDetails { field_order: None }), + ("LookAroundAssertion", StructDetails { field_order: Some(&[0, 2, 1]) }), ("BindingProperty", StructDetails { field_order: None }), - ("NullLiteral", StructDetails { field_order: None }), - ("SwitchStatement", StructDetails { field_order: None }), - ("IdentifierName", StructDetails { field_order: None }), - ("JSXOpeningFragment", StructDetails { field_order: None }), - ("TSInterfaceBody", StructDetails { field_order: None }), - ("TSIndexedAccessType", StructDetails { field_order: None }), - ("Modifiers", StructDetails { field_order: None }), - ("TemplateElement", StructDetails { field_order: None }), - ("JSXText", StructDetails { field_order: None }), - ("ExportAllDeclaration", StructDetails { field_order: None }), - ("TSMappedType", StructDetails { field_order: Some(&[0, 1, 2, 3, 5, 6, 4]) }), - ("TaggedTemplateExpression", StructDetails { field_order: None }), - ("TSNamedTupleMember", StructDetails { field_order: None }), - ("TSTypeLiteral", StructDetails { field_order: None }), - ("Character", StructDetails { field_order: Some(&[0, 2, 1]) }), - ("TSTupleType", StructDetails { field_order: None }), - ("AssignmentTargetPropertyProperty", StructDetails { field_order: None }), - ("TSThisType", StructDetails { field_order: None }), - ("TSTypeParameter", StructDetails { field_order: None }), + ("TSModuleDeclaration", StructDetails { field_order: Some(&[0, 1, 2, 4, 5, 3]) }), + ("JSXSpreadChild", StructDetails { field_order: None }), + ("Super", StructDetails { field_order: None }), + ("TSOptionalType", StructDetails { field_order: None }), + ("LabelIdentifier", StructDetails { field_order: None }), + ("ClassStringDisjunction", StructDetails { field_order: Some(&[0, 2, 1]) }), + ("ImportNamespaceSpecifier", StructDetails { field_order: None }), + ("BindingIdentifier", StructDetails { field_order: None }), ("TSAnyKeyword", StructDetails { field_order: None }), - ("TSStringKeyword", StructDetails { field_order: None }), - ("ForOfStatement", StructDetails { field_order: Some(&[0, 5, 1, 2, 3, 4]) }), - ("AssignmentTargetWithDefault", StructDetails { field_order: None }), - ("LabeledStatement", StructDetails { field_order: None }), - ("WithStatement", StructDetails { field_order: None }), - ("JSXExpressionContainer", StructDetails { field_order: None }), - ("ThrowStatement", StructDetails { field_order: None }), - ("IdentifierReference", StructDetails { field_order: None }), - ("Program", StructDetails { field_order: Some(&[0, 7, 1, 2, 3, 4, 5, 6]) }), - ("BoundaryAssertion", StructDetails { field_order: None }), - ("JSXEmptyExpression", StructDetails { field_order: None }), - ("JSXClosingFragment", StructDetails { field_order: None }), - ("TSNullKeyword", StructDetails { field_order: None }), - ("AssignmentPattern", StructDetails { field_order: None }), - ("Class", StructDetails { field_order: Some(&[0, 9, 1, 2, 3, 4, 5, 6, 7, 10, 11, 8]) }), - ("StaticImport", StructDetails { field_order: None }), - ("ForInStatement", StructDetails { field_order: None }), - ("NamedReference", StructDetails { field_order: None }), - ("JSXOpeningElement", StructDetails { field_order: None }), - ("Disjunction", StructDetails { field_order: None }), - ("ForStatement", StructDetails { field_order: None }), - ("ParenthesizedExpression", StructDetails { field_order: None }), - ("ContinueStatement", StructDetails { field_order: None }), - ("EmptyStatement", StructDetails { field_order: None }), - ("ArrayPattern", StructDetails { field_order: None }), - ("TSTypeOperator", StructDetails { field_order: Some(&[0, 2, 1]) }), - ("UpdateExpression", StructDetails { field_order: Some(&[0, 2, 3, 1]) }), - ("SourceType", StructDetails { field_order: None }), - ("ClassString", StructDetails { field_order: Some(&[0, 2, 1]) }), - ("ImportSpecifier", StructDetails { field_order: None }), - ("TSIntersectionType", StructDetails { field_order: None }), - ("ImportAttribute", StructDetails { field_order: None }), - ("TSNeverKeyword", StructDetails { field_order: None }), - ("ImportDefaultSpecifier", StructDetails { field_order: None }), + ("ClassBody", StructDetails { field_order: None }), + ("StaticExport", StructDetails { field_order: None }), + ("FormalParameters", StructDetails { field_order: Some(&[0, 3, 1, 2]) }), + ("ReturnStatement", StructDetails { field_order: None }), + ("BlockStatement", StructDetails { field_order: None }), + ("TSSatisfiesExpression", StructDetails { field_order: None }), + ("LogicalExpression", StructDetails { field_order: Some(&[0, 1, 3, 2]) }), ("CallExpression", StructDetails { field_order: None }), - ("PrivateIdentifier", StructDetails { field_order: None }), - ("TSUnionType", StructDetails { field_order: None }), - ("TSTypeReference", StructDetails { field_order: None }), - ("Error", StructDetails { field_order: Some(&[4, 0, 1, 2, 3]) }), - ("ArrayExpression", StructDetails { field_order: None }), - ("CommentNewlines", StructDetails { field_order: None }), - ("TSSymbolKeyword", StructDetails { field_order: None }), - ("TSInferType", StructDetails { field_order: None }), + ("FixedSizeAllocatorMetadata", StructDetails { field_order: Some(&[1, 0, 2]) }), + ("Token", StructDetails { field_order: None }), + ("TSLiteralType", StructDetails { field_order: None }), + ("Program", StructDetails { field_order: Some(&[0, 8, 1, 2, 3, 4, 5, 6, 7]) }), ("VariableDeclarator", StructDetails { field_order: Some(&[0, 3, 1, 2, 4]) }), - ("TemplateLiteral", StructDetails { field_order: None }), - ("PrivateInExpression", StructDetails { field_order: None }), - ("JSXFragment", StructDetails { field_order: None }), - ("ImportEntry", StructDetails { field_order: None }), - ("NonMaxU32", StructDetails { field_order: None }), - ("TSTypeAssertion", StructDetails { field_order: None }), - ( - "AccessorProperty", - StructDetails { field_order: Some(&[0, 5, 1, 2, 3, 4, 6, 7, 8, 9, 10]) }, - ), - ("TSTypePredicate", StructDetails { field_order: Some(&[0, 1, 3, 2]) }), - ("LabelIdentifier", StructDetails { field_order: None }), + ("TSConditionalType", StructDetails { field_order: None }), + ("TSExternalModuleReference", StructDetails { field_order: None }), + ("TSMappedType", StructDetails { field_order: Some(&[0, 1, 2, 3, 5, 6, 4]) }), + ("PrivateFieldExpression", StructDetails { field_order: None }), + ("ExportNamedDeclaration", StructDetails { field_order: Some(&[0, 1, 2, 3, 5, 4]) }), + ("TSTypeParameter", StructDetails { field_order: None }), + ("RawTransferData", StructDetails { field_order: None }), + ("CharacterClassRange", StructDetails { field_order: None }), + ("TSTypeReference", StructDetails { field_order: None }), + ("JSXOpeningFragment", StructDetails { field_order: None }), + ("BoundaryAssertion", StructDetails { field_order: None }), + ("ComputedMemberExpression", StructDetails { field_order: None }), ("TSEnumMember", StructDetails { field_order: None }), + ("UnicodePropertyEscape", StructDetails { field_order: Some(&[0, 3, 4, 1, 2]) }), + ("CharacterClassEscape", StructDetails { field_order: None }), + ("ParenthesizedExpression", StructDetails { field_order: None }), + ("ThisExpression", StructDetails { field_order: None }), + ("TSThisParameter", StructDetails { field_order: None }), + ("TSTypeParameterDeclaration", StructDetails { field_order: None }), ("CharacterClass", StructDetails { field_order: Some(&[0, 2, 3, 4, 1]) }), - ("TSImportTypeQualifiedName", StructDetails { field_order: None }), - ("JSXAttribute", StructDetails { field_order: None }), - ("TSClassImplements", StructDetails { field_order: None }), + ("TSGlobalDeclaration", StructDetails { field_order: Some(&[0, 1, 2, 4, 3]) }), + ("JSXClosingElement", StructDetails { field_order: None }), + ("Disjunction", StructDetails { field_order: None }), + ("IdentifierName", StructDetails { field_order: None }), + ("TSConstructSignatureDeclaration", StructDetails { field_order: None }), + ("Elision", StructDetails { field_order: None }), + ("BinaryExpression", StructDetails { field_order: Some(&[0, 1, 3, 2]) }), + ("NewExpression", StructDetails { field_order: None }), + ("TSVoidKeyword", StructDetails { field_order: None }), + ("JSXClosingFragment", StructDetails { field_order: None }), + ("JSDocNonNullableType", StructDetails { field_order: None }), + ("JSXElement", StructDetails { field_order: None }), ("BreakStatement", StructDetails { field_order: None }), - ("TSConstructorType", StructDetails { field_order: Some(&[0, 5, 1, 2, 3, 4]) }), - ("TSCallSignatureDeclaration", StructDetails { field_order: None }), - ("ExportDefaultDeclaration", StructDetails { field_order: None }), - ("Modifier", StructDetails { field_order: None }), - ("TSFunctionType", StructDetails { field_order: None }), + ("Decorator", StructDetails { field_order: None }), + ("TSNeverKeyword", StructDetails { field_order: None }), + ("TSImportTypeQualifiedName", StructDetails { field_order: None }), ("IgnoreGroup", StructDetails { field_order: None }), - ("StaticExport", StructDetails { field_order: None }), - ("MetaProperty", StructDetails { field_order: None }), - ("TSInterfaceHeritage", StructDetails { field_order: None }), - ("CharacterClassRange", StructDetails { field_order: None }), - ("LogicalExpression", StructDetails { field_order: Some(&[0, 1, 3, 2]) }), - ("TSIntrinsicKeyword", StructDetails { field_order: None }), - ("TSConditionalType", StructDetails { field_order: None }), + ("Alternative", StructDetails { field_order: None }), ("TSPropertySignature", StructDetails { field_order: Some(&[0, 3, 4, 5, 1, 2]) }), - ("TSQualifiedName", StructDetails { field_order: None }), - ("ImportExpression", StructDetails { field_order: None }), - ("TSRestType", StructDetails { field_order: None }), - ("Pattern", StructDetails { field_order: None }), - ("TSParenthesizedType", StructDetails { field_order: None }), - ("TSImportType", StructDetails { field_order: None }), - ("FormalParameter", StructDetails { field_order: None }), - ("TemplateElementValue", StructDetails { field_order: None }), - ("TSThisParameter", StructDetails { field_order: None }), + ("ImportAttribute", StructDetails { field_order: None }), ("RegExpLiteral", StructDetails { field_order: None }), - ("JSXSpreadAttribute", StructDetails { field_order: None }), - ("TSSatisfiesExpression", StructDetails { field_order: None }), - ("ClassBody", StructDetails { field_order: None }), - ("ClassStringDisjunction", StructDetails { field_order: Some(&[0, 2, 1]) }), - ("TSMethodSignature", StructDetails { field_order: Some(&[0, 1, 7, 8, 9, 2, 3, 4, 5, 6]) }), - ("WhileStatement", StructDetails { field_order: None }), - ("JSXIdentifier", StructDetails { field_order: None }), - ("TSTemplateLiteralType", StructDetails { field_order: None }), - ("Alternative", StructDetails { field_order: None }), + ("TSInstantiationExpression", StructDetails { field_order: None }), + ("JSXEmptyExpression", StructDetails { field_order: None }), + ("WithStatement", StructDetails { field_order: None }), + ("TSThisType", StructDetails { field_order: None }), + ("TSObjectKeyword", StructDetails { field_order: None }), + ("TSIntrinsicKeyword", StructDetails { field_order: None }), ("TSArrayType", StructDetails { field_order: None }), - ("PrivateFieldExpression", StructDetails { field_order: None }), - ("ExportNamedDeclaration", StructDetails { field_order: Some(&[0, 1, 2, 3, 5, 4]) }), - ("Directive", StructDetails { field_order: None }), - ("TryStatement", StructDetails { field_order: None }), + ("ExportEntry", StructDetails { field_order: Some(&[1, 0, 2, 3, 4, 5, 6]) }), + ("MetaProperty", StructDetails { field_order: None }), + ("TSUnionType", StructDetails { field_order: None }), + ("TSQualifiedName", StructDetails { field_order: None }), ("AwaitExpression", StructDetails { field_order: None }), - ("StaticBlock", StructDetails { field_order: None }), - ("TSTypeAliasDeclaration", StructDetails { field_order: Some(&[0, 1, 2, 3, 5, 4]) }), - ("VariableDeclaration", StructDetails { field_order: Some(&[0, 2, 1, 3]) }), + ("ForStatement", StructDetails { field_order: None }), + ("DoWhileStatement", StructDetails { field_order: None }), + ("TSTypeQuery", StructDetails { field_order: None }), + ("ThrowStatement", StructDetails { field_order: None }), + ("TSNullKeyword", StructDetails { field_order: None }), + ("SymbolId", StructDetails { field_order: None }), + ("YieldExpression", StructDetails { field_order: Some(&[0, 2, 1]) }), ("TSExportAssignment", StructDetails { field_order: None }), - ("JSDocNonNullableType", StructDetails { field_order: None }), - ("FixedSizeAllocatorMetadata", StructDetails { field_order: Some(&[1, 0, 2]) }), - ("TSNonNullExpression", StructDetails { field_order: None }), - ("BinaryExpression", StructDetails { field_order: Some(&[0, 1, 3, 2]) }), - ("TSNamespaceExportDeclaration", StructDetails { field_order: None }), + ("ArrayExpression", StructDetails { field_order: None }), + ("V8IntrinsicExpression", StructDetails { field_order: None }), + ("ArrayAssignmentTarget", StructDetails { field_order: None }), + ("ReferenceId", StructDetails { field_order: None }), + ("CommentNodeId", StructDetails { field_order: None }), + ("WithClause", StructDetails { field_order: Some(&[0, 2, 1]) }), + ("Class", StructDetails { field_order: Some(&[0, 9, 1, 2, 3, 4, 5, 6, 7, 10, 11, 8]) }), + ("TSMethodSignature", StructDetails { field_order: Some(&[0, 1, 7, 8, 9, 2, 3, 4, 5, 6]) }), + ("Directive", StructDetails { field_order: None }), + ("TSUnknownKeyword", StructDetails { field_order: None }), + ( + "ArrowFunctionExpression", + StructDetails { field_order: Some(&[0, 6, 7, 1, 2, 3, 4, 5, 8, 9]) }, + ), + ("AssignmentTargetWithDefault", StructDetails { field_order: None }), + ("RawTransferMetadata2", StructDetails { field_order: Some(&[1, 2, 0]) }), + ("ExportSpecifier", StructDetails { field_order: None }), + ("TSTypeLiteral", StructDetails { field_order: None }), + ("ContinueStatement", StructDetails { field_order: None }), + ("JSDocNullableType", StructDetails { field_order: None }), + ("TSTypeAssertion", StructDetails { field_order: None }), + ("JSXSpreadAttribute", StructDetails { field_order: None }), + ("NullLiteral", StructDetails { field_order: None }), + ("SwitchCase", StructDetails { field_order: None }), + ("RegExp", StructDetails { field_order: None }), + ("TSModuleBlock", StructDetails { field_order: None }), + ("StaticMemberExpression", StructDetails { field_order: None }), + ("CatchParameter", StructDetails { field_order: None }), ("BooleanLiteral", StructDetails { field_order: None }), + ("TSInterfaceHeritage", StructDetails { field_order: None }), + ("Character", StructDetails { field_order: Some(&[0, 2, 1]) }), + ("RegExpFlags", StructDetails { field_order: None }), + ("BigIntLiteral", StructDetails { field_order: None }), ("IndexedReference", StructDetails { field_order: None }), - ("FormalParameters", StructDetails { field_order: Some(&[0, 3, 1, 2]) }), + ("TSInferType", StructDetails { field_order: None }), + ("NameSpan", StructDetails { field_order: Some(&[1, 0]) }), + ("TSEnumDeclaration", StructDetails { field_order: Some(&[0, 1, 2, 4, 5, 3]) }), + ("TSStringKeyword", StructDetails { field_order: None }), + ("ImportDeclaration", StructDetails { field_order: Some(&[0, 1, 2, 4, 3, 5]) }), + ("StaticImport", StructDetails { field_order: None }), + ("TSIndexedAccessType", StructDetails { field_order: None }), ( - "Function", - StructDetails { field_order: Some(&[0, 8, 1, 9, 10, 11, 2, 3, 4, 5, 6, 7, 12, 13]) }, + "AccessorProperty", + StructDetails { field_order: Some(&[0, 5, 1, 2, 3, 4, 6, 7, 8, 9, 10]) }, ), - ("YieldExpression", StructDetails { field_order: Some(&[0, 2, 1]) }), - ("TSGlobalDeclaration", StructDetails { field_order: Some(&[0, 1, 2, 4, 3]) }), - ("TSIndexSignature", StructDetails { field_order: None }), - ("TSIndexSignatureName", StructDetails { field_order: None }), - ("Super", StructDetails { field_order: None }), - ("Quantifier", StructDetails { field_order: Some(&[0, 1, 2, 4, 3]) }), - ("ObjectPattern", StructDetails { field_order: None }), - ("DynamicImport", StructDetails { field_order: None }), - ("Decorator", StructDetails { field_order: None }), + ("TSAsExpression", StructDetails { field_order: None }), + ("RegExpPattern", StructDetails { field_order: None }), + ("IfStatement", StructDetails { field_order: None }), ("TSEnumBody", StructDetails { field_order: None }), - ("NameSpan", StructDetails { field_order: Some(&[1, 0]) }), - ("ComputedMemberExpression", StructDetails { field_order: None }), - ("TSEnumDeclaration", StructDetails { field_order: Some(&[0, 1, 2, 4, 5, 3]) }), - ("TSInstantiationExpression", StructDetails { field_order: None }), - ("JSXMemberExpression", StructDetails { field_order: None }), + ("EcmaScriptModule", StructDetails { field_order: Some(&[4, 0, 1, 2, 3]) }), + ("JSXAttribute", StructDetails { field_order: None }), + ("CommentNewlines", StructDetails { field_order: None }), + ("TSNumberKeyword", StructDetails { field_order: None }), + ("VariableDeclaration", StructDetails { field_order: Some(&[0, 2, 1, 3]) }), + ("TemplateLiteral", StructDetails { field_order: None }), + ("TSFunctionType", StructDetails { field_order: None }), + ("FunctionBody", StructDetails { field_order: None }), + ("TSIndexSignature", StructDetails { field_order: None }), + ("Modifier", StructDetails { field_order: None }), + ("ObjectProperty", StructDetails { field_order: Some(&[0, 3, 1, 2, 4, 5, 6]) }), + ("JSXOpeningElement", StructDetails { field_order: None }), + ("ChainExpression", StructDetails { field_order: None }), + ("JSXIdentifier", StructDetails { field_order: None }), + ("TSTypeParameterInstantiation", StructDetails { field_order: None }), + ("RawTransferMetadata", StructDetails { field_order: Some(&[1, 2, 0]) }), + ("StringLiteral", StructDetails { field_order: None }), + ("UpdateExpression", StructDetails { field_order: Some(&[0, 2, 3, 1]) }), + ("Dot", StructDetails { field_order: None }), + ("AssignmentTargetPropertyIdentifier", StructDetails { field_order: None }), + ("BindingPattern", StructDetails { field_order: None }), + ("ScopeId", StructDetails { field_order: None }), + ("TSBigIntKeyword", StructDetails { field_order: None }), + ("Span", StructDetails { field_order: None }), + ("ImportDefaultSpecifier", StructDetails { field_order: None }), + ("JSXText", StructDetails { field_order: None }), + ("ForOfStatement", StructDetails { field_order: Some(&[0, 5, 1, 2, 3, 4]) }), + ("UnaryExpression", StructDetails { field_order: Some(&[0, 2, 1]) }), + ("ClassString", StructDetails { field_order: Some(&[0, 2, 1]) }), + ("JSXFragment", StructDetails { field_order: None }), + ("SourceType", StructDetails { field_order: None }), + ("SpreadElement", StructDetails { field_order: None }), + ("TSConstructorType", StructDetails { field_order: Some(&[0, 5, 1, 2, 3, 4]) }), + ("AssignmentTargetPropertyProperty", StructDetails { field_order: None }), + ("TSCallSignatureDeclaration", StructDetails { field_order: None }), + ("TSTypePredicate", StructDetails { field_order: Some(&[0, 1, 3, 2]) }), ("TSImportEqualsDeclaration", StructDetails { field_order: None }), - ("TSVoidKeyword", StructDetails { field_order: None }), - ("SequenceExpression", StructDetails { field_order: None }), - ("ExportSpecifier", StructDetails { field_order: None }), - ("TSTypeParameterDeclaration", StructDetails { field_order: None }), - ("BindingIdentifier", StructDetails { field_order: None }), - ("JSXClosingElement", StructDetails { field_order: None }), - ("CommentNodeId", StructDetails { field_order: None }), - ("TSExternalModuleReference", StructDetails { field_order: None }), - ("JSXElement", StructDetails { field_order: None }), - ("ExpressionStatement", StructDetails { field_order: None }), - ("ReferenceId", StructDetails { field_order: None }), + ("NonMaxU32", StructDetails { field_order: None }), + ("ImportEntry", StructDetails { field_order: None }), + ("WhileStatement", StructDetails { field_order: None }), + ("ObjectExpression", StructDetails { field_order: None }), + ("Pattern", StructDetails { field_order: None }), + ("TemplateElement", StructDetails { field_order: None }), ], }; diff --git a/crates/oxc_ast_visit/src/utf8_to_utf16/mod.rs b/crates/oxc_ast_visit/src/utf8_to_utf16/mod.rs index 406254520df4b..241e68b199d06 100644 --- a/crates/oxc_ast_visit/src/utf8_to_utf16/mod.rs +++ b/crates/oxc_ast_visit/src/utf8_to_utf16/mod.rs @@ -139,6 +139,7 @@ mod test { SourceType::default(), ";'🤨' // 🤨", ast.vec1(Comment::new(8, 15, CommentKind::Line)), + ast.vec(), None, ast.vec(), ast.vec_from_array([ diff --git a/crates/oxc_formatter/src/ast_nodes/generated/ast_nodes.rs b/crates/oxc_formatter/src/ast_nodes/generated/ast_nodes.rs index f06e0408163ab..ae02425fe7bcb 100644 --- a/crates/oxc_formatter/src/ast_nodes/generated/ast_nodes.rs +++ b/crates/oxc_formatter/src/ast_nodes/generated/ast_nodes.rs @@ -809,6 +809,11 @@ impl<'a> AstNode<'a, Program<'a>> { &self.inner.comments } + #[inline] + pub fn tokens(&self) -> &Vec<'a, Token<'a>> { + &self.inner.tokens + } + #[inline] pub fn hashbang(&self) -> Option<&AstNode<'a, Hashbang<'a>>> { let following_span = self diff --git a/crates/oxc_isolated_declarations/src/lib.rs b/crates/oxc_isolated_declarations/src/lib.rs index 96c5ef9636166..fca73fca60b10 100644 --- a/crates/oxc_isolated_declarations/src/lib.rs +++ b/crates/oxc_isolated_declarations/src/lib.rs @@ -88,11 +88,13 @@ impl<'a> IsolatedDeclarations<'a> { let source_type = SourceType::d_ts(); let directives = self.ast.vec(); let stmts = self.transform_program(program); + let tokens = self.ast.vec(); let program = self.ast.program( SPAN, source_type, program.source_text, self.ast.vec_from_iter(program.comments.iter().copied()), + tokens, None, directives, stmts, diff --git a/crates/oxc_parser/src/lexer/kind.rs b/crates/oxc_parser/src/lexer/kind.rs index f2b5eb1a32400..568ae6b5bc855 100644 --- a/crates/oxc_parser/src/lexer/kind.rs +++ b/crates/oxc_parser/src/lexer/kind.rs @@ -207,6 +207,28 @@ pub enum Kind { use Kind::*; impl Kind { + /// TODO: investigate performance of doing this on the JS side + pub fn to_tseslint_type(self) -> Option<&'static str> { + let ty = match self { + Eof | Undetermined | Skip => return None, + True | False => "Boolean", + Null => "Null", + Ident | Constructor | Undefined | As | Any | Satisfies | Type | String | Number => { + "Identifier" + } + PrivateIdentifier => "PrivateIdentifier", + JSXText => "JSXText", + RegExp => "RegularExpression", + Str => "String", + NoSubstitutionTemplate | TemplateHead | TemplateMiddle | TemplateTail => "Template", + Decimal | Float | Binary | Octal | Hex | PositiveExponential | NegativeExponential + | DecimalBigInt | BinaryBigInt | OctalBigInt | HexBigInt => "Numeric", + _ if self.is_any_keyword() => "Keyword", + _ => "Punctuator", + }; + Some(ty) + } + #[inline] pub fn is_eof(self) -> bool { self == Eof diff --git a/crates/oxc_parser/src/lib.rs b/crates/oxc_parser/src/lib.rs index ed49486a9ee7e..7a2b319e1cf97 100644 --- a/crates/oxc_parser/src/lib.rs +++ b/crates/oxc_parser/src/lib.rs @@ -90,7 +90,7 @@ pub mod lexer; use oxc_allocator::{Allocator, Box as ArenaBox, Dummy, Vec as ArenaVec}; use oxc_ast::{ - AstBuilder, + AstBuilder, ast, ast::{Expression, Program}, }; use oxc_diagnostics::OxcDiagnostic; @@ -521,7 +521,8 @@ impl<'a> ParserImpl<'a> { return Err(vec![error]); } self.check_unfinished_errors(); - let errors = self.lexer.errors.into_iter().chain(self.errors).collect::>(); + let errors: Vec = + self.lexer.errors.into_iter().chain(self.errors).collect::>(); if !errors.is_empty() { return Err(errors); } @@ -540,11 +541,20 @@ impl<'a> ParserImpl<'a> { let span = Span::new(0, self.source_text.len() as u32); let comments = self.ast.vec_from_iter(self.lexer.trivia_builder.comments.iter().copied()); + let tokens = self.ast.vec_from_iter(self.lexer.tokens().iter().filter_map(|token| { + token.kind().to_tseslint_type().map(|ty| ast::Token { + span: token.span(), + r#type: self.ast.atom(ty), + flags: None, + pattern: None, + }) + })); self.ast.program( span, self.source_type, self.source_text, comments, + tokens, hashbang, directives, statements, diff --git a/crates/oxc_traverse/src/generated/ancestor.rs b/crates/oxc_traverse/src/generated/ancestor.rs index 6af9de0b78f1e..4e8a793a989c7 100644 --- a/crates/oxc_traverse/src/generated/ancestor.rs +++ b/crates/oxc_traverse/src/generated/ancestor.rs @@ -2515,6 +2515,7 @@ pub(crate) const OFFSET_PROGRAM_SPAN: usize = offset_of!(Program, span); pub(crate) const OFFSET_PROGRAM_SOURCE_TYPE: usize = offset_of!(Program, source_type); pub(crate) const OFFSET_PROGRAM_SOURCE_TEXT: usize = offset_of!(Program, source_text); pub(crate) const OFFSET_PROGRAM_COMMENTS: usize = offset_of!(Program, comments); +pub(crate) const OFFSET_PROGRAM_TOKENS: usize = offset_of!(Program, tokens); pub(crate) const OFFSET_PROGRAM_HASHBANG: usize = offset_of!(Program, hashbang); pub(crate) const OFFSET_PROGRAM_DIRECTIVES: usize = offset_of!(Program, directives); pub(crate) const OFFSET_PROGRAM_BODY: usize = offset_of!(Program, body); @@ -2548,6 +2549,11 @@ impl<'a, 't> ProgramWithoutHashbang<'a, 't> { unsafe { &*((self.0 as *const u8).add(OFFSET_PROGRAM_COMMENTS) as *const Vec<'a, Comment>) } } + #[inline] + pub fn tokens(self) -> &'t Vec<'a, Token<'a>> { + unsafe { &*((self.0 as *const u8).add(OFFSET_PROGRAM_TOKENS) as *const Vec<'a, Token<'a>>) } + } + #[inline] pub fn directives(self) -> &'t Vec<'a, Directive<'a>> { unsafe { @@ -2606,6 +2612,11 @@ impl<'a, 't> ProgramWithoutDirectives<'a, 't> { unsafe { &*((self.0 as *const u8).add(OFFSET_PROGRAM_COMMENTS) as *const Vec<'a, Comment>) } } + #[inline] + pub fn tokens(self) -> &'t Vec<'a, Token<'a>> { + unsafe { &*((self.0 as *const u8).add(OFFSET_PROGRAM_TOKENS) as *const Vec<'a, Token<'a>>) } + } + #[inline] pub fn hashbang(self) -> &'t Option> { unsafe { @@ -2663,6 +2674,11 @@ impl<'a, 't> ProgramWithoutBody<'a, 't> { unsafe { &*((self.0 as *const u8).add(OFFSET_PROGRAM_COMMENTS) as *const Vec<'a, Comment>) } } + #[inline] + pub fn tokens(self) -> &'t Vec<'a, Token<'a>> { + unsafe { &*((self.0 as *const u8).add(OFFSET_PROGRAM_TOKENS) as *const Vec<'a, Token<'a>>) } + } + #[inline] pub fn hashbang(self) -> &'t Option> { unsafe { diff --git a/napi/parser/src-js/generated/deserialize/js.js b/napi/parser/src-js/generated/deserialize/js.js index a55851ca70bfb..8b72f88bd4a21 100644 --- a/napi/parser/src-js/generated/deserialize/js.js +++ b/napi/parser/src-js/generated/deserialize/js.js @@ -34,13 +34,13 @@ function deserializeProgram(pos) { program = { type: "Program", body: null, - sourceType: deserializeModuleKind(pos + 125), + sourceType: deserializeModuleKind(pos + 149), hashbang: null, start, end, }; - program.hashbang = deserializeOptionHashbang(pos + 48); - (program.body = deserializeVecDirective(pos + 72)).push(...deserializeVecStatement(pos + 96)); + program.hashbang = deserializeOptionHashbang(pos + 72); + (program.body = deserializeVecDirective(pos + 96)).push(...deserializeVecStatement(pos + 120)); return program; } @@ -4229,6 +4229,19 @@ function deserializeComment(pos) { }; } +function deserializeToken(pos) { + let start = deserializeU32(pos), + end = deserializeU32(pos + 4); + return { + type: deserializeStr(pos + 8), + flags: deserializeOptionStr(pos + 24), + pattern: deserializeOptionStr(pos + 40), + value: sourceText.slice(start, end), + start, + end, + }; +} + function deserializeNameSpan(pos) { let start = deserializeU32(pos), end = deserializeU32(pos + 4); @@ -4553,9 +4566,10 @@ function deserializeModuleKind(pos) { function deserializeRawTransferData(pos) { return { program: deserializeProgram(pos), - comments: deserializeVecComment(pos + 128), - module: deserializeEcmaScriptModule(pos + 152), - errors: deserializeVecError(pos + 256), + comments: deserializeVecComment(pos + 152), + tokens: deserializeVecToken(pos + 176), + module: deserializeEcmaScriptModule(pos + 200), + errors: deserializeVecError(pos + 304), }; } @@ -4667,6 +4681,18 @@ function deserializeVecComment(pos) { return arr; } +function deserializeVecToken(pos) { + let arr = [], + pos32 = pos >> 2; + pos = uint32[pos32]; + let endPos = pos + uint32[pos32 + 2] * 56; + for (; pos !== endPos; ) { + arr.push(deserializeToken(pos)); + pos += 56; + } + return arr; +} + function deserializeOptionHashbang(pos) { if (uint32[(pos + 8) >> 2] === 0 && uint32[(pos + 12) >> 2] === 0) return null; return deserializeHashbang(pos); diff --git a/napi/parser/src-js/generated/deserialize/js_parent.js b/napi/parser/src-js/generated/deserialize/js_parent.js index b5e417456deac..16709444d2623 100644 --- a/napi/parser/src-js/generated/deserialize/js_parent.js +++ b/napi/parser/src-js/generated/deserialize/js_parent.js @@ -40,14 +40,14 @@ function deserializeProgram(pos) { program = (parent = { type: "Program", body: null, - sourceType: deserializeModuleKind(pos + 125), + sourceType: deserializeModuleKind(pos + 149), hashbang: null, start, end, parent: null, }); - program.hashbang = deserializeOptionHashbang(pos + 48); - (program.body = deserializeVecDirective(pos + 72)).push(...deserializeVecStatement(pos + 96)); + program.hashbang = deserializeOptionHashbang(pos + 72); + (program.body = deserializeVecDirective(pos + 96)).push(...deserializeVecStatement(pos + 120)); parent = null; return program; } @@ -4962,6 +4962,19 @@ function deserializeComment(pos) { }; } +function deserializeToken(pos) { + let start = deserializeU32(pos), + end = deserializeU32(pos + 4); + return { + type: deserializeStr(pos + 8), + flags: deserializeOptionStr(pos + 24), + pattern: deserializeOptionStr(pos + 40), + value: sourceText.slice(start, end), + start, + end, + }; +} + function deserializeNameSpan(pos) { let start = deserializeU32(pos), end = deserializeU32(pos + 4); @@ -5286,9 +5299,10 @@ function deserializeModuleKind(pos) { function deserializeRawTransferData(pos) { return { program: deserializeProgram(pos), - comments: deserializeVecComment(pos + 128), - module: deserializeEcmaScriptModule(pos + 152), - errors: deserializeVecError(pos + 256), + comments: deserializeVecComment(pos + 152), + tokens: deserializeVecToken(pos + 176), + module: deserializeEcmaScriptModule(pos + 200), + errors: deserializeVecError(pos + 304), }; } @@ -5400,6 +5414,18 @@ function deserializeVecComment(pos) { return arr; } +function deserializeVecToken(pos) { + let arr = [], + pos32 = pos >> 2; + pos = uint32[pos32]; + let endPos = pos + uint32[pos32 + 2] * 56; + for (; pos !== endPos; ) { + arr.push(deserializeToken(pos)); + pos += 56; + } + return arr; +} + function deserializeOptionHashbang(pos) { if (uint32[(pos + 8) >> 2] === 0 && uint32[(pos + 12) >> 2] === 0) return null; return deserializeHashbang(pos); diff --git a/napi/parser/src-js/generated/deserialize/js_range.js b/napi/parser/src-js/generated/deserialize/js_range.js index 0ee4434619a8d..cb0ebb289dbda 100644 --- a/napi/parser/src-js/generated/deserialize/js_range.js +++ b/napi/parser/src-js/generated/deserialize/js_range.js @@ -34,14 +34,14 @@ function deserializeProgram(pos) { program = { type: "Program", body: null, - sourceType: deserializeModuleKind(pos + 125), + sourceType: deserializeModuleKind(pos + 149), hashbang: null, start, end, range: [start, end], }; - program.hashbang = deserializeOptionHashbang(pos + 48); - (program.body = deserializeVecDirective(pos + 72)).push(...deserializeVecStatement(pos + 96)); + program.hashbang = deserializeOptionHashbang(pos + 72); + (program.body = deserializeVecDirective(pos + 96)).push(...deserializeVecStatement(pos + 120)); return program; } @@ -4676,6 +4676,20 @@ function deserializeComment(pos) { }; } +function deserializeToken(pos) { + let start = deserializeU32(pos), + end = deserializeU32(pos + 4); + return { + type: deserializeStr(pos + 8), + flags: deserializeOptionStr(pos + 24), + pattern: deserializeOptionStr(pos + 40), + value: sourceText.slice(start, end), + start, + end, + range: [start, end], + }; +} + function deserializeNameSpan(pos) { let start = deserializeU32(pos), end = deserializeU32(pos + 4); @@ -5016,9 +5030,10 @@ function deserializeModuleKind(pos) { function deserializeRawTransferData(pos) { return { program: deserializeProgram(pos), - comments: deserializeVecComment(pos + 128), - module: deserializeEcmaScriptModule(pos + 152), - errors: deserializeVecError(pos + 256), + comments: deserializeVecComment(pos + 152), + tokens: deserializeVecToken(pos + 176), + module: deserializeEcmaScriptModule(pos + 200), + errors: deserializeVecError(pos + 304), }; } @@ -5133,6 +5148,18 @@ function deserializeVecComment(pos) { return arr; } +function deserializeVecToken(pos) { + let arr = [], + pos32 = pos >> 2; + pos = uint32[pos32]; + let endPos = pos + uint32[pos32 + 2] * 56; + for (; pos !== endPos; ) { + arr.push(deserializeToken(pos)); + pos += 56; + } + return arr; +} + function deserializeOptionHashbang(pos) { if (uint32[(pos + 8) >> 2] === 0 && uint32[(pos + 12) >> 2] === 0) return null; return deserializeHashbang(pos); diff --git a/napi/parser/src-js/generated/deserialize/js_range_parent.js b/napi/parser/src-js/generated/deserialize/js_range_parent.js index fada9f950a0be..cebd525b58a7f 100644 --- a/napi/parser/src-js/generated/deserialize/js_range_parent.js +++ b/napi/parser/src-js/generated/deserialize/js_range_parent.js @@ -40,15 +40,15 @@ function deserializeProgram(pos) { program = (parent = { type: "Program", body: null, - sourceType: deserializeModuleKind(pos + 125), + sourceType: deserializeModuleKind(pos + 149), hashbang: null, start, end, range: [start, end], parent: null, }); - program.hashbang = deserializeOptionHashbang(pos + 48); - (program.body = deserializeVecDirective(pos + 72)).push(...deserializeVecStatement(pos + 96)); + program.hashbang = deserializeOptionHashbang(pos + 72); + (program.body = deserializeVecDirective(pos + 96)).push(...deserializeVecStatement(pos + 120)); parent = null; return program; } @@ -5210,6 +5210,20 @@ function deserializeComment(pos) { }; } +function deserializeToken(pos) { + let start = deserializeU32(pos), + end = deserializeU32(pos + 4); + return { + type: deserializeStr(pos + 8), + flags: deserializeOptionStr(pos + 24), + pattern: deserializeOptionStr(pos + 40), + value: sourceText.slice(start, end), + start, + end, + range: [start, end], + }; +} + function deserializeNameSpan(pos) { let start = deserializeU32(pos), end = deserializeU32(pos + 4); @@ -5550,9 +5564,10 @@ function deserializeModuleKind(pos) { function deserializeRawTransferData(pos) { return { program: deserializeProgram(pos), - comments: deserializeVecComment(pos + 128), - module: deserializeEcmaScriptModule(pos + 152), - errors: deserializeVecError(pos + 256), + comments: deserializeVecComment(pos + 152), + tokens: deserializeVecToken(pos + 176), + module: deserializeEcmaScriptModule(pos + 200), + errors: deserializeVecError(pos + 304), }; } @@ -5667,6 +5682,18 @@ function deserializeVecComment(pos) { return arr; } +function deserializeVecToken(pos) { + let arr = [], + pos32 = pos >> 2; + pos = uint32[pos32]; + let endPos = pos + uint32[pos32 + 2] * 56; + for (; pos !== endPos; ) { + arr.push(deserializeToken(pos)); + pos += 56; + } + return arr; +} + function deserializeOptionHashbang(pos) { if (uint32[(pos + 8) >> 2] === 0 && uint32[(pos + 12) >> 2] === 0) return null; return deserializeHashbang(pos); diff --git a/napi/parser/src-js/generated/deserialize/ts.js b/napi/parser/src-js/generated/deserialize/ts.js index 982a94b7060b7..1f7b95402eb9b 100644 --- a/napi/parser/src-js/generated/deserialize/ts.js +++ b/napi/parser/src-js/generated/deserialize/ts.js @@ -33,14 +33,14 @@ function deserializeProgram(pos) { program = { type: "Program", body: null, - sourceType: deserializeModuleKind(pos + 125), + sourceType: deserializeModuleKind(pos + 149), hashbang: null, start: 0, end, }; - program.hashbang = deserializeOptionHashbang(pos + 48); - let body = (program.body = deserializeVecDirective(pos + 72)); - body.push(...deserializeVecStatement(pos + 96)); + program.hashbang = deserializeOptionHashbang(pos + 72); + let body = (program.body = deserializeVecDirective(pos + 96)); + body.push(...deserializeVecStatement(pos + 120)); { let start; if (body.length > 0) { @@ -4482,6 +4482,19 @@ function deserializeComment(pos) { }; } +function deserializeToken(pos) { + let start = deserializeU32(pos), + end = deserializeU32(pos + 4); + return { + type: deserializeStr(pos + 8), + flags: deserializeOptionStr(pos + 24), + pattern: deserializeOptionStr(pos + 40), + value: sourceText.slice(start, end), + start, + end, + }; +} + function deserializeNameSpan(pos) { let start = deserializeU32(pos), end = deserializeU32(pos + 4); @@ -4806,9 +4819,10 @@ function deserializeModuleKind(pos) { function deserializeRawTransferData(pos) { return { program: deserializeProgram(pos), - comments: deserializeVecComment(pos + 128), - module: deserializeEcmaScriptModule(pos + 152), - errors: deserializeVecError(pos + 256), + comments: deserializeVecComment(pos + 152), + tokens: deserializeVecToken(pos + 176), + module: deserializeEcmaScriptModule(pos + 200), + errors: deserializeVecError(pos + 304), }; } @@ -4920,6 +4934,18 @@ function deserializeVecComment(pos) { return arr; } +function deserializeVecToken(pos) { + let arr = [], + pos32 = pos >> 2; + pos = uint32[pos32]; + let endPos = pos + uint32[pos32 + 2] * 56; + for (; pos !== endPos; ) { + arr.push(deserializeToken(pos)); + pos += 56; + } + return arr; +} + function deserializeOptionHashbang(pos) { if (uint32[(pos + 8) >> 2] === 0 && uint32[(pos + 12) >> 2] === 0) return null; return deserializeHashbang(pos); diff --git a/napi/parser/src-js/generated/deserialize/ts_parent.js b/napi/parser/src-js/generated/deserialize/ts_parent.js index 49ac6cf63f6eb..cbdf6caa50351 100644 --- a/napi/parser/src-js/generated/deserialize/ts_parent.js +++ b/napi/parser/src-js/generated/deserialize/ts_parent.js @@ -39,15 +39,15 @@ function deserializeProgram(pos) { program = (parent = { type: "Program", body: null, - sourceType: deserializeModuleKind(pos + 125), + sourceType: deserializeModuleKind(pos + 149), hashbang: null, start: 0, end, parent: null, }); - program.hashbang = deserializeOptionHashbang(pos + 48); - let body = (program.body = deserializeVecDirective(pos + 72)); - body.push(...deserializeVecStatement(pos + 96)); + program.hashbang = deserializeOptionHashbang(pos + 72); + let body = (program.body = deserializeVecDirective(pos + 96)); + body.push(...deserializeVecStatement(pos + 120)); { let start; if (body.length > 0) { @@ -5223,6 +5223,19 @@ function deserializeComment(pos) { }; } +function deserializeToken(pos) { + let start = deserializeU32(pos), + end = deserializeU32(pos + 4); + return { + type: deserializeStr(pos + 8), + flags: deserializeOptionStr(pos + 24), + pattern: deserializeOptionStr(pos + 40), + value: sourceText.slice(start, end), + start, + end, + }; +} + function deserializeNameSpan(pos) { let start = deserializeU32(pos), end = deserializeU32(pos + 4); @@ -5547,9 +5560,10 @@ function deserializeModuleKind(pos) { function deserializeRawTransferData(pos) { return { program: deserializeProgram(pos), - comments: deserializeVecComment(pos + 128), - module: deserializeEcmaScriptModule(pos + 152), - errors: deserializeVecError(pos + 256), + comments: deserializeVecComment(pos + 152), + tokens: deserializeVecToken(pos + 176), + module: deserializeEcmaScriptModule(pos + 200), + errors: deserializeVecError(pos + 304), }; } @@ -5661,6 +5675,18 @@ function deserializeVecComment(pos) { return arr; } +function deserializeVecToken(pos) { + let arr = [], + pos32 = pos >> 2; + pos = uint32[pos32]; + let endPos = pos + uint32[pos32 + 2] * 56; + for (; pos !== endPos; ) { + arr.push(deserializeToken(pos)); + pos += 56; + } + return arr; +} + function deserializeOptionHashbang(pos) { if (uint32[(pos + 8) >> 2] === 0 && uint32[(pos + 12) >> 2] === 0) return null; return deserializeHashbang(pos); diff --git a/napi/parser/src-js/generated/deserialize/ts_range.js b/napi/parser/src-js/generated/deserialize/ts_range.js index 6a39bfa15a011..688abcc97af0f 100644 --- a/napi/parser/src-js/generated/deserialize/ts_range.js +++ b/napi/parser/src-js/generated/deserialize/ts_range.js @@ -33,15 +33,15 @@ function deserializeProgram(pos) { program = { type: "Program", body: null, - sourceType: deserializeModuleKind(pos + 125), + sourceType: deserializeModuleKind(pos + 149), hashbang: null, start: 0, end, range: [0, end], }; - program.hashbang = deserializeOptionHashbang(pos + 48); - let body = (program.body = deserializeVecDirective(pos + 72)); - body.push(...deserializeVecStatement(pos + 96)); + program.hashbang = deserializeOptionHashbang(pos + 72); + let body = (program.body = deserializeVecDirective(pos + 96)); + body.push(...deserializeVecStatement(pos + 120)); { let start; if (body.length > 0) { @@ -4928,6 +4928,20 @@ function deserializeComment(pos) { }; } +function deserializeToken(pos) { + let start = deserializeU32(pos), + end = deserializeU32(pos + 4); + return { + type: deserializeStr(pos + 8), + flags: deserializeOptionStr(pos + 24), + pattern: deserializeOptionStr(pos + 40), + value: sourceText.slice(start, end), + start, + end, + range: [start, end], + }; +} + function deserializeNameSpan(pos) { let start = deserializeU32(pos), end = deserializeU32(pos + 4); @@ -5268,9 +5282,10 @@ function deserializeModuleKind(pos) { function deserializeRawTransferData(pos) { return { program: deserializeProgram(pos), - comments: deserializeVecComment(pos + 128), - module: deserializeEcmaScriptModule(pos + 152), - errors: deserializeVecError(pos + 256), + comments: deserializeVecComment(pos + 152), + tokens: deserializeVecToken(pos + 176), + module: deserializeEcmaScriptModule(pos + 200), + errors: deserializeVecError(pos + 304), }; } @@ -5385,6 +5400,18 @@ function deserializeVecComment(pos) { return arr; } +function deserializeVecToken(pos) { + let arr = [], + pos32 = pos >> 2; + pos = uint32[pos32]; + let endPos = pos + uint32[pos32 + 2] * 56; + for (; pos !== endPos; ) { + arr.push(deserializeToken(pos)); + pos += 56; + } + return arr; +} + function deserializeOptionHashbang(pos) { if (uint32[(pos + 8) >> 2] === 0 && uint32[(pos + 12) >> 2] === 0) return null; return deserializeHashbang(pos); diff --git a/napi/parser/src-js/generated/deserialize/ts_range_parent.js b/napi/parser/src-js/generated/deserialize/ts_range_parent.js index 145dfb2a649ae..78addb49c17ba 100644 --- a/napi/parser/src-js/generated/deserialize/ts_range_parent.js +++ b/napi/parser/src-js/generated/deserialize/ts_range_parent.js @@ -39,16 +39,16 @@ function deserializeProgram(pos) { program = (parent = { type: "Program", body: null, - sourceType: deserializeModuleKind(pos + 125), + sourceType: deserializeModuleKind(pos + 149), hashbang: null, start: 0, end, range: [0, end], parent: null, }); - program.hashbang = deserializeOptionHashbang(pos + 48); - let body = (program.body = deserializeVecDirective(pos + 72)); - body.push(...deserializeVecStatement(pos + 96)); + program.hashbang = deserializeOptionHashbang(pos + 72); + let body = (program.body = deserializeVecDirective(pos + 96)); + body.push(...deserializeVecStatement(pos + 120)); { let start; if (body.length > 0) { @@ -5478,6 +5478,20 @@ function deserializeComment(pos) { }; } +function deserializeToken(pos) { + let start = deserializeU32(pos), + end = deserializeU32(pos + 4); + return { + type: deserializeStr(pos + 8), + flags: deserializeOptionStr(pos + 24), + pattern: deserializeOptionStr(pos + 40), + value: sourceText.slice(start, end), + start, + end, + range: [start, end], + }; +} + function deserializeNameSpan(pos) { let start = deserializeU32(pos), end = deserializeU32(pos + 4); @@ -5818,9 +5832,10 @@ function deserializeModuleKind(pos) { function deserializeRawTransferData(pos) { return { program: deserializeProgram(pos), - comments: deserializeVecComment(pos + 128), - module: deserializeEcmaScriptModule(pos + 152), - errors: deserializeVecError(pos + 256), + comments: deserializeVecComment(pos + 152), + tokens: deserializeVecToken(pos + 176), + module: deserializeEcmaScriptModule(pos + 200), + errors: deserializeVecError(pos + 304), }; } @@ -5935,6 +5950,18 @@ function deserializeVecComment(pos) { return arr; } +function deserializeVecToken(pos) { + let arr = [], + pos32 = pos >> 2; + pos = uint32[pos32]; + let endPos = pos + uint32[pos32 + 2] * 56; + for (; pos !== endPos; ) { + arr.push(deserializeToken(pos)); + pos += 56; + } + return arr; +} + function deserializeOptionHashbang(pos) { if (uint32[(pos + 8) >> 2] === 0 && uint32[(pos + 12) >> 2] === 0) return null; return deserializeHashbang(pos); diff --git a/napi/parser/src-js/generated/lazy/constructors.js b/napi/parser/src-js/generated/lazy/constructors.js index 59684976a8f02..b2ab262a2524c 100644 --- a/napi/parser/src-js/generated/lazy/constructors.js +++ b/napi/parser/src-js/generated/lazy/constructors.js @@ -36,19 +36,19 @@ export class Program { get sourceType() { const internal = this.#internal; - return new SourceType(internal.pos + 124, internal.ast); + return new SourceType(internal.pos + 148, internal.ast); } get hashbang() { const internal = this.#internal; - return constructOptionHashbang(internal.pos + 48, internal.ast); + return constructOptionHashbang(internal.pos + 72, internal.ast); } get body() { const internal = this.#internal, cached = internal.$body; if (cached !== void 0) return cached; - return (internal.$body = constructVecStatement(internal.pos + 96, internal.ast)); + return (internal.$body = constructVecStatement(internal.pos + 120, internal.ast)); } toJSON() { @@ -11794,6 +11794,68 @@ export class Comment { const DebugComment = class Comment {}; +export class Token { + #internal; + + constructor(pos, ast) { + if (ast?.token !== TOKEN) constructorError(); + + const { nodes } = ast; + const cached = nodes.get(pos); + if (cached !== void 0) return cached; + + this.#internal = { pos, ast, $type: void 0, $flags: void 0, $pattern: void 0 }; + nodes.set(pos, this); + } + + get start() { + const internal = this.#internal; + return constructU32(internal.pos, internal.ast); + } + + get end() { + const internal = this.#internal; + return constructU32(internal.pos + 4, internal.ast); + } + + get type() { + const internal = this.#internal, + cached = internal.$type; + if (cached !== void 0) return cached; + return (internal.$type = constructStr(internal.pos + 8, internal.ast)); + } + + get flags() { + const internal = this.#internal, + cached = internal.$flags; + if (cached !== void 0) return cached; + return (internal.$flags = constructOptionStr(internal.pos + 24, internal.ast)); + } + + get pattern() { + const internal = this.#internal, + cached = internal.$pattern; + if (cached !== void 0) return cached; + return (internal.$pattern = constructOptionStr(internal.pos + 40, internal.ast)); + } + + toJSON() { + return { + start: this.start, + end: this.end, + type: this.type, + flags: this.flags, + pattern: this.pattern, + }; + } + + [inspectSymbol]() { + return Object.setPrototypeOf(this.toJSON(), DebugToken.prototype); + } +} + +const DebugToken = class Token {}; + export class NameSpan { #internal; @@ -12277,7 +12339,7 @@ export class RawTransferData { const cached = nodes.get(pos + 1); if (cached !== void 0) return cached; - this.#internal = { pos, ast, $comments: void 0, $errors: void 0 }; + this.#internal = { pos, ast, $comments: void 0, $tokens: void 0, $errors: void 0 }; nodes.set(pos + 1, this); } @@ -12290,25 +12352,33 @@ export class RawTransferData { const internal = this.#internal, cached = internal.$comments; if (cached !== void 0) return cached; - return (internal.$comments = constructVecComment(internal.pos + 128, internal.ast)); + return (internal.$comments = constructVecComment(internal.pos + 152, internal.ast)); + } + + get tokens() { + const internal = this.#internal, + cached = internal.$tokens; + if (cached !== void 0) return cached; + return (internal.$tokens = constructVecToken(internal.pos + 176, internal.ast)); } get module() { const internal = this.#internal; - return new EcmaScriptModule(internal.pos + 152, internal.ast); + return new EcmaScriptModule(internal.pos + 200, internal.ast); } get errors() { const internal = this.#internal, cached = internal.$errors; if (cached !== void 0) return cached; - return (internal.$errors = constructVecError(internal.pos + 256, internal.ast)); + return (internal.$errors = constructVecError(internal.pos + 304, internal.ast)); } toJSON() { return { program: this.program, comments: this.comments, + tokens: this.tokens, module: this.module, errors: this.errors, }; @@ -12669,6 +12739,16 @@ function constructComment(pos, ast) { return new Comment(pos, ast); } +function constructVecToken(pos, ast) { + const { uint32 } = ast.buffer, + pos32 = pos >> 2; + return new NodeArray(uint32[pos32], uint32[pos32 + 2], 56, constructToken, ast); +} + +function constructToken(pos, ast) { + return new Token(pos, ast); +} + function constructOptionHashbang(pos, ast) { if (ast.buffer.uint32[(pos + 8) >> 2] === 0 && ast.buffer.uint32[(pos + 12) >> 2] === 0) return null; diff --git a/napi/parser/src-js/generated/lazy/walk.js b/napi/parser/src-js/generated/lazy/walk.js index 1a1904eadbddf..dd2019b3f55ae 100644 --- a/napi/parser/src-js/generated/lazy/walk.js +++ b/napi/parser/src-js/generated/lazy/walk.js @@ -197,8 +197,8 @@ function walkProgram(pos, ast, visitors) { if (enter !== null) enter(node); } - walkOptionHashbang(pos + 48, ast, visitors); - walkVecStatement(pos + 96, ast, visitors); + walkOptionHashbang(pos + 72, ast, visitors); + walkVecStatement(pos + 120, ast, visitors); if (exit !== null) exit(node); } diff --git a/napi/parser/src/generated/assert_layouts.rs b/napi/parser/src/generated/assert_layouts.rs index c272146b3d0eb..7abae0535ce9e 100644 --- a/napi/parser/src/generated/assert_layouts.rs +++ b/napi/parser/src/generated/assert_layouts.rs @@ -10,12 +10,13 @@ use crate::raw_transfer_types::*; #[cfg(target_pointer_width = "64")] const _: () = { // Padding: 0 bytes - assert!(size_of::() == 280); + assert!(size_of::() == 328); assert!(align_of::() == 8); assert!(offset_of!(RawTransferData, program) == 0); - assert!(offset_of!(RawTransferData, comments) == 128); - assert!(offset_of!(RawTransferData, module) == 152); - assert!(offset_of!(RawTransferData, errors) == 256); + assert!(offset_of!(RawTransferData, comments) == 152); + assert!(offset_of!(RawTransferData, tokens) == 176); + assert!(offset_of!(RawTransferData, module) == 200); + assert!(offset_of!(RawTransferData, errors) == 304); // Padding: 3 bytes assert!(size_of::() == 16); @@ -68,12 +69,13 @@ const _: () = { #[cfg(target_pointer_width = "32")] const _: () = if cfg!(target_family = "wasm") || align_of::() == 8 { // Padding: 0 bytes - assert!(size_of::() == 188); + assert!(size_of::() == 220); assert!(align_of::() == 4); assert!(offset_of!(RawTransferData, program) == 0); - assert!(offset_of!(RawTransferData, comments) == 88); - assert!(offset_of!(RawTransferData, module) == 104); - assert!(offset_of!(RawTransferData, errors) == 172); + assert!(offset_of!(RawTransferData, comments) == 104); + assert!(offset_of!(RawTransferData, tokens) == 120); + assert!(offset_of!(RawTransferData, module) == 136); + assert!(offset_of!(RawTransferData, errors) == 204); // Padding: 3 bytes assert!(size_of::() == 16); diff --git a/napi/parser/src/generated/derive_estree.rs b/napi/parser/src/generated/derive_estree.rs index 459891860c6c9..02f94c007307b 100644 --- a/napi/parser/src/generated/derive_estree.rs +++ b/napi/parser/src/generated/derive_estree.rs @@ -14,6 +14,7 @@ impl ESTree for RawTransferData<'_> { let mut state = serializer.serialize_struct(); state.serialize_field("program", &self.program); state.serialize_field("comments", &self.comments); + state.serialize_field("tokens", &self.tokens); state.serialize_field("module", &self.module); state.serialize_field("errors", &self.errors); state.end(); diff --git a/napi/parser/src/raw_transfer.rs b/napi/parser/src/raw_transfer.rs index 6b106346939f3..8120162466726 100644 --- a/napi/parser/src/raw_transfer.rs +++ b/napi/parser/src/raw_transfer.rs @@ -12,8 +12,10 @@ use napi_derive::napi; use oxc::{ allocator::{Allocator, FromIn, Vec as ArenaVec}, + ast::ast::Token, ast_visit::utf8_to_utf16::Utf8ToUtf16, semantic::SemanticBuilder, + span::Atom, }; use oxc_napi::get_source_type; @@ -225,6 +227,7 @@ unsafe fn parse_raw_impl( let ret = parse_impl(&allocator, source_type, source_text, &options); let mut program = ret.program; let mut comments = mem::replace(&mut program.comments, ArenaVec::new_in(&allocator)); + let tokens = ret.tokens; let mut module_record = ret.module_record; // Convert errors. @@ -264,11 +267,27 @@ unsafe fn parse_raw_impl( } } + let tokens: ArenaVec<'_, Token<'_>> = if let Some(tokens) = tokens { + ArenaVec::from_iter_in( + tokens.into_iter().filter_map(|token| { + token.kind().to_tseslint_type().map(|ty| Token { + span: token.span(), + r#type: Atom::from_in(ty, &allocator), + flags: None, + pattern: None, + }) + }), + &allocator, + ) + } else { + ArenaVec::new_in(&allocator) + }; + // Convert module record let module = EcmaScriptModule::from_in(module_record, &allocator); // Write `RawTransferData` to arena, and return pointer to it - let data = RawTransferData { program, comments, module, errors }; + let data = RawTransferData { program, comments, tokens, module, errors }; let data = allocator.alloc(data); ptr::from_ref(data).cast::() }; diff --git a/napi/parser/src/raw_transfer_types.rs b/napi/parser/src/raw_transfer_types.rs index a58dc19590f4b..7f2a7aa0eb791 100644 --- a/napi/parser/src/raw_transfer_types.rs +++ b/napi/parser/src/raw_transfer_types.rs @@ -6,7 +6,7 @@ use rustc_hash::FxHashMap; use oxc::{ allocator::{Allocator, FromIn, Vec}, - ast::ast::{Comment, Program}, + ast::ast::{Comment, Program, Token}, diagnostics::{LabeledSpan, NamedSource, OxcDiagnostic, Severity}, span::{Atom, Span, format_atom}, syntax::module_record::{DynamicImport, ExportEntry, ImportEntry, ModuleRecord, NameSpan}, @@ -21,6 +21,7 @@ use oxc_estree::ESTree; pub struct RawTransferData<'a> { pub program: Program<'a>, pub comments: Vec<'a, Comment>, + pub tokens: Vec<'a, Token<'a>>, pub module: EcmaScriptModule<'a>, pub errors: Vec<'a, Error<'a>>, } diff --git a/tasks/ast_tools/src/generators/raw_transfer.rs b/tasks/ast_tools/src/generators/raw_transfer.rs index 5acb34e7348f8..0f545cd3847d2 100644 --- a/tasks/ast_tools/src/generators/raw_transfer.rs +++ b/tasks/ast_tools/src/generators/raw_transfer.rs @@ -134,10 +134,6 @@ fn generate_deserializers( #[rustfmt::skip] let mut code = format!(" - /* IF LINTER */ - import {{ tokens, initTokens }} from '../plugins/tokens.js'; - /* END_IF */ - let uint8, uint32, float64, sourceText, sourceIsAscii, sourceByteLen; let astId = 0; diff --git a/tasks/ast_tools/src/generators/utf8_to_utf16.rs b/tasks/ast_tools/src/generators/utf8_to_utf16.rs index 1012c1e2e7a96..9d4173149dfc5 100644 --- a/tasks/ast_tools/src/generators/utf8_to_utf16.rs +++ b/tasks/ast_tools/src/generators/utf8_to_utf16.rs @@ -50,6 +50,7 @@ fn generate(schema: &Schema, codegen: &Codegen) -> TokenStream { let estree_derive_id = codegen.get_derive_id_by_name("ESTree"); let span_type_id = schema.type_names["Span"]; let comment_type_id = schema.type_names["Comment"]; + let token_type_id = schema.type_names["Token"]; // Types with custom visitors (see comment above) let custom_visitor_type_ids = [ @@ -76,6 +77,10 @@ fn generate(schema: &Schema, codegen: &Codegen) -> TokenStream { return None; } + if struct_def.id == token_type_id { + return None; + } + if !struct_def.generates_derive(estree_derive_id) { return None; } diff --git a/tasks/ast_tools/src/main.rs b/tasks/ast_tools/src/main.rs index 98dd28dc68cb4..c79f93ab92406 100644 --- a/tasks/ast_tools/src/main.rs +++ b/tasks/ast_tools/src/main.rs @@ -216,6 +216,7 @@ static SOURCE_PATHS: &[&str] = &[ "crates/oxc_ast/src/ast/jsx.rs", "crates/oxc_ast/src/ast/ts.rs", "crates/oxc_ast/src/ast/comment.rs", + "crates/oxc_ast/src/ast/token.rs", "crates/oxc_ast/src/serialize/mod.rs", "crates/oxc_ast/src/serialize/basic.rs", "crates/oxc_ast/src/serialize/literal.rs",