diff --git a/apps/oxlint/src-js/generated/deserialize.js b/apps/oxlint/src-js/generated/deserialize.js index 5766e348f68c3..38769c56179c5 100644 --- a/apps/oxlint/src-js/generated/deserialize.js +++ b/apps/oxlint/src-js/generated/deserialize.js @@ -1,6 +1,8 @@ // Auto-generated code, DO NOT EDIT DIRECTLY! // To edit this generated file you have to edit `tasks/ast_tools/src/generators/raw_transfer.rs`. +import { tokens, initTokens } from '../plugins/tokens.js'; + let uint8, uint32, float64, @@ -62,6 +64,10 @@ function deserializeProgram(pos) { Object.defineProperty(this, 'comments', { value: comments }); return comments; }, + get tokens() { + tokens === null && initTokens(); + return tokens; + }, start: 0, end, range: [0, end], diff --git a/apps/oxlint/src-js/generated/types.d.ts b/apps/oxlint/src-js/generated/types.d.ts index 3900dfb843453..eae2ef6acf7a7 100644 --- a/apps/oxlint/src-js/generated/types.d.ts +++ b/apps/oxlint/src-js/generated/types.d.ts @@ -2,8 +2,9 @@ // To edit this generated file you have to edit `tasks/ast_tools/src/generators/typescript.rs`. import { Span } from '../plugins/location.ts'; +import { Token } from '../plugins/tokens.ts'; import { Comment } from '../plugins/types.ts'; -export { Span, Comment }; +export { Span, Comment, Token }; export interface Program extends Span { type: 'Program'; @@ -11,6 +12,7 @@ export interface Program extends Span { sourceType: ModuleKind; hashbang: Hashbang | null; comments: Comment[]; + tokens: Token[]; parent: null; } diff --git a/apps/oxlint/src-js/plugins/tokens.ts b/apps/oxlint/src-js/plugins/tokens.ts index 474c248391026..db5f92cb40b38 100644 --- a/apps/oxlint/src-js/plugins/tokens.ts +++ b/apps/oxlint/src-js/plugins/tokens.ts @@ -135,7 +135,7 @@ export interface TemplateToken extends BaseToken { // Tokens for the current file parsed by TS-ESLint. // Created lazily only when needed. -let tokens: Token[] | null = null; +export let tokens: Token[] | null = null; let comments: CommentToken[] | null = null; let tokensWithComments: Token[] | null = null; @@ -146,8 +146,10 @@ let tsEslintParse: typeof import('@typescript-eslint/typescript-estree').parse | /** * Initialize TS-ESLint tokens for current file. + * + * Caller must ensure `sourceText` is initialized before calling this function. */ -function initTokens() { +export function initTokens() { debugAssertIsNonNull(sourceText); // Lazy-load TS-ESLint. diff --git a/apps/oxlint/test/fixtures/tokens/.oxlintrc.json b/apps/oxlint/test/fixtures/tokens/.oxlintrc.json new file mode 100644 index 0000000000000..82c90b0946545 --- /dev/null +++ b/apps/oxlint/test/fixtures/tokens/.oxlintrc.json @@ -0,0 +1,9 @@ +{ + "jsPlugins": ["./plugin.ts"], + "categories": { + "correctness": "off" + }, + "rules": { + "tokens-plugin/tokens": "error" + } +} diff --git a/apps/oxlint/test/fixtures/tokens/files/index.js b/apps/oxlint/test/fixtures/tokens/files/index.js new file mode 100644 index 0000000000000..a8d69af2c9822 --- /dev/null +++ b/apps/oxlint/test/fixtures/tokens/files/index.js @@ -0,0 +1,8 @@ +// Leading comment + +let x = /* inline comment */ 1; + +// Another comment +let y = 2; + +// Trailing comment diff --git a/apps/oxlint/test/fixtures/tokens/output.snap.md b/apps/oxlint/test/fixtures/tokens/output.snap.md new file mode 100644 index 0000000000000..dd43a8ea50182 --- /dev/null +++ b/apps/oxlint/test/fixtures/tokens/output.snap.md @@ -0,0 +1,36 @@ +# Exit code +1 + +# stdout +``` + x tokens-plugin(tokens): Tokens: + | Keyword loc=3:0-3:3 range=20-23 "let" + | Identifier loc=3:4-3:5 range=24-25 "x" + | Punctuator loc=3:6-3:7 range=26-27 "=" + | Numeric loc=3:29-3:30 range=49-50 "1" + | Punctuator loc=3:30-3:31 range=50-51 ";" + | Keyword loc=6:0-6:3 range=72-75 "let" + | Identifier loc=6:4-6:5 range=76-77 "y" + | Punctuator loc=6:6-6:7 range=78-79 "=" + | Numeric loc=6:8-6:9 range=80-81 "2" + | Punctuator loc=6:9-6:10 range=81-82 ";" + ,-[files/index.js:1:1] + 1 | ,-> // Leading comment + 2 | | + 3 | | let x = /* inline comment */ 1; + 4 | | + 5 | | // Another comment + 6 | | let y = 2; + 7 | | + 8 | `-> // Trailing comment + `---- + +Found 0 warnings and 1 error. +Finished in Xms on 1 file using X threads. +``` + +# stderr +``` +WARNING: JS plugins are experimental and not subject to semver. +Breaking changes are possible while JS plugins support is under development. +``` diff --git a/apps/oxlint/test/fixtures/tokens/plugin.ts b/apps/oxlint/test/fixtures/tokens/plugin.ts new file mode 100644 index 0000000000000..8f88427d72026 --- /dev/null +++ b/apps/oxlint/test/fixtures/tokens/plugin.ts @@ -0,0 +1,33 @@ +import type { Plugin, Rule } from '../../../dist/index.js'; + +const rule: Rule = { + create(context) { + const { sourceCode } = context, + { ast } = sourceCode; + + // Note: Comments should not appear in `ast.tokens` + context.report({ + message: + `Tokens:\n` + + ast.tokens + .map( + ({ type, loc, range, value }) => + `${type.padEnd(17)} ` + + `loc=${loc.start.line}:${loc.start.column}-${loc.end.line}:${loc.end.column} `.padEnd(16) + + `range=${range[0]}-${range[1]} `.padEnd(10) + + `"${value}"`, + ) + .join('\n'), + node: { range: [0, sourceCode.text.length] }, + }); + + return {}; + }, +}; + +const plugin: Plugin = { + meta: { name: 'tokens-plugin' }, + rules: { tokens: rule }, +}; + +export default plugin; diff --git a/crates/oxc_ast/src/serialize/mod.rs b/crates/oxc_ast/src/serialize/mod.rs index 622897c5d9c14..a2b1970cd1134 100644 --- a/crates/oxc_ast/src/serialize/mod.rs +++ b/crates/oxc_ast/src/serialize/mod.rs @@ -142,6 +142,12 @@ impl Program<'_> { return comments; }, /* END_IF */ + /* IF LINTER */ + get tokens() { + if (tokens === null) initTokens(); + return tokens; + }, + /* END_IF */ start, end, ...(RANGE && { range: [start, end] }), diff --git a/tasks/ast_tools/src/generators/raw_transfer.rs b/tasks/ast_tools/src/generators/raw_transfer.rs index f84959e2eee65..41604d51cc595 100644 --- a/tasks/ast_tools/src/generators/raw_transfer.rs +++ b/tasks/ast_tools/src/generators/raw_transfer.rs @@ -134,6 +134,10 @@ fn generate_deserializers( #[rustfmt::skip] let mut code = format!(" + /* IF LINTER */ + import {{ tokens, initTokens }} from '../plugins/tokens.js'; + /* END_IF */ + let uint8, uint32, float64, sourceText, sourceIsAscii, sourceByteLen; let astId = 0; diff --git a/tasks/ast_tools/src/generators/typescript.rs b/tasks/ast_tools/src/generators/typescript.rs index 7d928fcc76d04..bf6b2ee99f7e3 100644 --- a/tasks/ast_tools/src/generators/typescript.rs +++ b/tasks/ast_tools/src/generators/typescript.rs @@ -479,11 +479,11 @@ fn amend_oxlint_types(code: &str) -> String { let mut code = SPAN_REGEX.replace(code, SpanReplacer).into_owned(); - // Add `comments` field to `Program` + // Add `comments` and `tokens` fields to `Program` #[expect(clippy::items_after_statements)] const HASHBANG_FIELD: &str = "hashbang: Hashbang | null;"; let index = code.find(HASHBANG_FIELD).unwrap(); - code.insert_str(index + HASHBANG_FIELD.len(), "comments: Comment[];"); + code.insert_str(index + HASHBANG_FIELD.len(), "comments: Comment[]; tokens: Token[];"); // Make `parent` fields non-optional #[expect(clippy::disallowed_methods)] @@ -492,8 +492,9 @@ fn amend_oxlint_types(code: &str) -> String { #[rustfmt::skip] code.insert_str(0, " import { Span } from '../plugins/location.ts'; + import { Token } from '../plugins/tokens.ts'; import { Comment } from '../plugins/types.ts'; - export { Span, Comment }; + export { Span, Comment, Token }; ");