Skip to content

feat(linter/plugins): serialize rust-parsed tokens #17025

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
lilnasy wants to merge 1 commit into from
Closed

feat(linter/plugins): serialize rust-parsed tokens #17025

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 34 additions & 7 deletions apps/oxlint/src-js/generated/deserialize.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
// Auto-generated code, DO NOT EDIT DIRECTLY!
// To edit this generated file you have to edit `tasks/ast_tools/src/generators/raw_transfer.rs`.

import { tokens, initTokens } from "../plugins/tokens.js";

let uint8,
uint32,
float64,
Expand Down Expand Up @@ -53,7 +51,7 @@ function deserializeProgram(pos) {
__proto__: NodeProto,
type: "Program",
body: null,
sourceType: deserializeModuleKind(pos + 125),
sourceType: deserializeModuleKind(pos + 149),
hashbang: null,
get comments() {
// Check AST in buffer is still the same AST (buffers are reused)
Expand All @@ -66,17 +64,19 @@ function deserializeProgram(pos) {
return comments;
},
get tokens() {
tokens === null && initTokens();
if (localAstId !== astId) throw Error("Tokens are only accessible while linting the file");
let tokens = deserializeVecToken(pos + 48);
Object.defineProperty(this, "tokens", { value: tokens });
return tokens;
},
start: 0,
end,
range: [0, end],
parent: null,
});
program.hashbang = deserializeOptionHashbang(pos + 48);
let body = (program.body = deserializeVecDirective(pos + 72));
body.push(...deserializeVecStatement(pos + 96));
program.hashbang = deserializeOptionHashbang(pos + 72);
let body = (program.body = deserializeVecDirective(pos + 96));
body.push(...deserializeVecStatement(pos + 120));
{
let start;
if (body.length > 0) {
Expand Down Expand Up @@ -5672,6 +5672,21 @@ function deserializeComment(pos) {
};
}

function deserializeToken(pos) {
let start = deserializeU32(pos),
end = deserializeU32(pos + 4);
return {
__proto__: NodeProto,
type: deserializeStr(pos + 8),
flags: deserializeOptionStr(pos + 24),
pattern: deserializeOptionStr(pos + 40),
value: sourceText.slice(start, end),
start,
end,
range: [start, end],
};
}

function deserializeAssignmentOperator(pos) {
switch (uint8[pos]) {
case 0:
Expand Down Expand Up @@ -5862,6 +5877,18 @@ function deserializeVecComment(pos) {
return arr;
}

function deserializeVecToken(pos) {
let arr = [],
pos32 = pos >> 2;
pos = uint32[pos32];
let endPos = pos + uint32[pos32 + 2] * 56;
for (; pos !== endPos; ) {
arr.push(deserializeToken(pos));
pos += 56;
}
return arr;
}

function deserializeOptionHashbang(pos) {
if (uint32[(pos + 8) >> 2] === 0 && uint32[(pos + 12) >> 2] === 0) return null;
return deserializeHashbang(pos);
Expand Down
21 changes: 12 additions & 9 deletions apps/oxlint/src-js/plugins/tokens.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,12 @@
* `SourceCode` methods related to tokens.
*/

import { ast, initAst } from "./source_code.ts";
import { parseTokens } from "./tokens_parse.ts";
import { ast, initAst, sourceText } from "./source_code.ts";
import { debugAssert, debugAssertIsNonNull } from "../utils/asserts.ts";

import type { Comment, Node, NodeOrToken } from "./types.ts";
import type { Span } from "./location.ts";
import { filePath } from "./context.ts";

/**
* Options for various `SourceCode` methods e.g. `getFirstToken`.
Expand Down Expand Up @@ -132,16 +132,19 @@ let comments: Comment[] | null = null;
export let tokensAndComments: TokenOrComment[] | null = null;

/**
* Initialize TS-ESLint tokens for current file.
*
* Caller must ensure `filePath` and `sourceText` are initialized before calling this function.
*/
export function initTokens() {
// Use TypeScript parser to get tokens
tokens = parseTokens();
debugAssertIsNonNull(filePath);
debugAssertIsNonNull(sourceText);

if (ast === null) initAst();
debugAssertIsNonNull(ast);

tokens = ast.tokens;

// Check `tokens` have valid ranges and are in ascending order
debugCheckValidRanges(tokens, "token");
// debugCheckValidRanges(tokens, "token");
}

/**
Expand Down Expand Up @@ -178,7 +181,7 @@ export function initTokensAndComments() {
debugAssertIsNonNull(ast);
comments = ast.comments;

debugCheckValidRanges(comments, "comment");
// debugCheckValidRanges(comments, "comment");
}

// Fast paths for file with no comments, or file which is only comments
Expand Down Expand Up @@ -289,7 +292,7 @@ function debugCheckTokensAndComments() {
}
}

debugCheckValidRanges(tokensAndComments, "token/comment");
// debugCheckValidRanges(tokensAndComments, "token/comment");
}

/**
Expand Down
18 changes: 17 additions & 1 deletion apps/oxlint/test/fixtures/getNodeByRangeIndex/output.snap.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,14 @@
6 |
`----

x getNode-plugin(getNode): type: TemplateElement
,-[files/index.ts:5:11]
4 |
5 | `___${123}___`;
: ^
6 |
`----

x getNode-plugin(getNode): type: TemplateElement
,-[files/index.ts:5:14]
4 |
Expand Down Expand Up @@ -314,6 +322,14 @@
8 |
`----

x getNode-plugin(getNode): type: TemplateElement
,-[files/index.ts:7:20]
6 |
7 | type T = `___${123}___`;
: ^
8 |
`----

x getNode-plugin(getNode): type: TemplateElement
,-[files/index.ts:7:23]
6 |
Expand Down Expand Up @@ -355,7 +371,7 @@
9 | // Comment
`----

Found 0 warnings and 45 errors.
Found 0 warnings and 47 errors.
Finished in Xms on 1 file using X threads.
```

Expand Down
2 changes: 2 additions & 0 deletions crates/oxc_ast/src/ast/js.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,8 @@ pub struct Program<'a> {
#[content_eq(skip)]
#[estree(skip)]
pub comments: Vec<'a, Comment>,
#[estree(skip)]
pub tokens: Vec<'a, Token<'a>>,
Comment on lines +60 to +61
Copy link
Member

@overlookmotel overlookmotel Dec 18, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we avoid adding this field to Program? Tokens can be returned in ParserReturn instead.

pub hashbang: Option<Hashbang<'a>>,
#[estree(prepend_to = body)]
pub directives: Vec<'a, Directive<'a>>,
Expand Down
2 changes: 2 additions & 0 deletions crates/oxc_ast/src/ast/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -187,12 +187,14 @@ pub(crate) mod js;
pub(crate) mod jsx;
pub(crate) mod literal;
pub(crate) mod macros;
pub(crate) mod token;
pub(crate) mod ts;

pub use comment::*;
pub use js::*;
pub use jsx::*;
pub use literal::*;
pub use token::*;
pub use ts::*;

use macros::inherit_variants;
27 changes: 27 additions & 0 deletions crates/oxc_ast/src/ast/token.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
use oxc_allocator::CloneIn;
use oxc_ast_macros::{ast, ast_meta};
use oxc_estree::ESTree;
use oxc_span::{Atom, ContentEq, GetSpan, GetSpanMut, Span};

#[ast]
#[generate_derive(CloneIn, ContentEq, ESTree, GetSpan, GetSpanMut)]
#[estree(add_fields(value = TokenValue), no_type, no_ts_def, no_parent)]
#[derive(Debug)]
/// Represents a token in the source code.
pub struct Token<'a> {
/// Span.
#[span]
pub span: Span,
/// Type.
pub r#type: Atom<'a>,
/// Flags.
pub flags: Option<Atom<'a>>,
/// Pattern.
pub pattern: Option<Atom<'a>>,
}

/// Custom deserializer for `value` field of `Token`.
#[ast_meta]
#[generate_derive(CloneIn, ContentEq, ESTree)]
#[estree(ts_type = "string", raw_deser = "SOURCE_TEXT.slice(THIS.start, THIS.end)")]
pub struct TokenValue<'a, 'b>(pub &'b Token<'a>);
Comment on lines +1 to +27
Copy link
Member

@overlookmotel overlookmotel Dec 18, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We shouldn't need this. We want JS side code to read the original Vec<Token> directly, without converting it.

Instead:

  • Add #[ast] attribute to the existing Token struct (and remove #[repr(transparent)] attr - #[ast] macro adds it automatically).
  • Add #[ast] #[generate_derive(ESTree)] #[estree(rename = "TokenKind")] attributes to the existing Kind (token kind) enum.
  • Implement ESTree trait on Token.
  • Add crates/oxc_parser/src/lexer/token.rs and crates/oxc_parser/src/lexer/kind.rs to the list of files that codegen processes at in ast_tools.
  • Add another type #[ast] struct Tokens<'a>(Vec<'a, Tokens>); in same file. We don't need to use that type, but adding it should make ast_tools generate a deserializeVecToken function.
  • Add a TOKEN flag to FLAG_NAMES in raw_transfer.rs generator.
  • Export deserializeVecToken from deserializer when TOKEN flag is enabled.

ESTree impl on Token will need to be manually defined (including the #[estree(raw_deser)] attr). Shout if you have any difficulty with this. Writing raw_deser implementations is pretty dreadful - my fault, it's terribly designed - and you'll need to write the byte offsets manually (since Token doesn't have real fields).

But you should be able to use deserializeTokenKind which codegen will generate, and also use the generated ESTree impl for Kind.

The conversion from Kind (token kind) to ESTree token type is the tricky part. Should be able to get codegen to do most of the work by adding #[estree(rename = "Keyword")] (etc) to all the variants of Kind.

No doubt, you didn't need me to explain every one of the steps above. I've just included them for clarity, to (hopefully) save you a little time. Please excuse me if I'm "teaching grandpa to suck eggs".

I've laid a bit of groundwork for this in #17050 and #17052.

Copy link
Member

@overlookmotel overlookmotel Dec 18, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh actually maybe we can get codegen to generate everything for us by:

  • Don't put #[ast] attr on Token.
  • Instead, add this:
/// Dummy type to communicate the content of `Token` to `oxc_ast_tools`.
#[ast(foreign = Token)]
#[generate_derive(ESTree)]
#[expect(dead_code)]
struct TokenAlias {
  pub span: Span,
  pub kind: Kind,
  #[estree(skip)]
  pub _align: U128Align,
}

/// Zero-sized type which has alignment of `u128`
#[repr(transparent)]
struct U128Align([u128; 0]);

This tells the codegen "treat Token as if it's defined like this".

You'd need to add a "special case" to codegen for U128Align (search ast_tools for PointerAlign).

EDIT: No this is a bad idea. Need to implement ESTree and raw_deser manually to handle the special case of extra fields on regexp tokens.

42 changes: 30 additions & 12 deletions crates/oxc_ast/src/generated/assert_layouts.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,16 +10,17 @@ use crate::ast::*;
#[cfg(target_pointer_width = "64")]
const _: () = {
// Padding: 1 bytes
assert!(size_of::<Program>() == 128);
assert!(size_of::<Program>() == 152);
assert!(align_of::<Program>() == 8);
assert!(offset_of!(Program, span) == 0);
assert!(offset_of!(Program, source_type) == 124);
assert!(offset_of!(Program, source_type) == 148);
assert!(offset_of!(Program, source_text) == 8);
assert!(offset_of!(Program, comments) == 24);
assert!(offset_of!(Program, hashbang) == 48);
assert!(offset_of!(Program, directives) == 72);
assert!(offset_of!(Program, body) == 96);
assert!(offset_of!(Program, scope_id) == 120);
assert!(offset_of!(Program, tokens) == 48);
assert!(offset_of!(Program, hashbang) == 72);
assert!(offset_of!(Program, directives) == 96);
assert!(offset_of!(Program, body) == 120);
assert!(offset_of!(Program, scope_id) == 144);

assert!(size_of::<Expression>() == 16);
assert!(align_of::<Expression>() == 8);
Expand Down Expand Up @@ -1621,21 +1622,30 @@ const _: () = {
assert!(offset_of!(Comment, position) == 13);
assert!(offset_of!(Comment, newlines) == 14);
assert!(offset_of!(Comment, content) == 15);

// Padding: 0 bytes
assert!(size_of::<Token>() == 56);
assert!(align_of::<Token>() == 8);
assert!(offset_of!(Token, span) == 0);
assert!(offset_of!(Token, r#type) == 8);
assert!(offset_of!(Token, flags) == 24);
assert!(offset_of!(Token, pattern) == 40);
};

#[cfg(target_pointer_width = "32")]
const _: () = if cfg!(target_family = "wasm") || align_of::<u64>() == 8 {
// Padding: 1 bytes
assert!(size_of::<Program>() == 88);
assert!(size_of::<Program>() == 104);
assert!(align_of::<Program>() == 4);
assert!(offset_of!(Program, span) == 0);
assert!(offset_of!(Program, source_type) == 84);
assert!(offset_of!(Program, source_type) == 100);
assert!(offset_of!(Program, source_text) == 8);
assert!(offset_of!(Program, comments) == 16);
assert!(offset_of!(Program, hashbang) == 32);
assert!(offset_of!(Program, directives) == 48);
assert!(offset_of!(Program, body) == 64);
assert!(offset_of!(Program, scope_id) == 80);
assert!(offset_of!(Program, tokens) == 32);
assert!(offset_of!(Program, hashbang) == 48);
assert!(offset_of!(Program, directives) == 64);
assert!(offset_of!(Program, body) == 80);
assert!(offset_of!(Program, scope_id) == 96);

assert!(size_of::<Expression>() == 8);
assert!(align_of::<Expression>() == 4);
Expand Down Expand Up @@ -3237,6 +3247,14 @@ const _: () = if cfg!(target_family = "wasm") || align_of::<u64>() == 8 {
assert!(offset_of!(Comment, position) == 13);
assert!(offset_of!(Comment, newlines) == 14);
assert!(offset_of!(Comment, content) == 15);

// Padding: 0 bytes
assert!(size_of::<Token>() == 32);
assert!(align_of::<Token>() == 4);
assert!(offset_of!(Token, span) == 0);
assert!(offset_of!(Token, r#type) == 8);
assert!(offset_of!(Token, flags) == 16);
assert!(offset_of!(Token, pattern) == 24);
};

#[cfg(not(any(target_pointer_width = "64", target_pointer_width = "32")))]
Expand Down
6 changes: 6 additions & 0 deletions crates/oxc_ast/src/generated/ast_builder.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ impl<'a> AstBuilder<'a> {
/// * `source_type`
/// * `source_text`
/// * `comments`: Sorted comments
/// * `tokens`
/// * `hashbang`
/// * `directives`
/// * `body`
Expand All @@ -37,6 +38,7 @@ impl<'a> AstBuilder<'a> {
source_type: SourceType,
source_text: &'a str,
comments: Vec<'a, Comment>,
tokens: Vec<'a, Token<'a>>,
hashbang: Option<Hashbang<'a>>,
directives: Vec<'a, Directive<'a>>,
body: Vec<'a, Statement<'a>>,
Expand All @@ -46,6 +48,7 @@ impl<'a> AstBuilder<'a> {
source_type,
source_text,
comments,
tokens,
hashbang,
directives,
body,
Expand All @@ -60,6 +63,7 @@ impl<'a> AstBuilder<'a> {
/// * `source_type`
/// * `source_text`
/// * `comments`: Sorted comments
/// * `tokens`
/// * `hashbang`
/// * `directives`
/// * `body`
Expand All @@ -71,6 +75,7 @@ impl<'a> AstBuilder<'a> {
source_type: SourceType,
source_text: &'a str,
comments: Vec<'a, Comment>,
tokens: Vec<'a, Token<'a>>,
hashbang: Option<Hashbang<'a>>,
directives: Vec<'a, Directive<'a>>,
body: Vec<'a, Statement<'a>>,
Expand All @@ -81,6 +86,7 @@ impl<'a> AstBuilder<'a> {
source_type,
source_text,
comments,
tokens,
hashbang,
directives,
body,
Expand Down
Loading
Loading