diff --git a/apps/oxlint/src-js/plugins/source_code.ts b/apps/oxlint/src-js/plugins/source_code.ts index 51c971b02b5e2..89d006b132f98 100644 --- a/apps/oxlint/src-js/plugins/source_code.ts +++ b/apps/oxlint/src-js/plugins/source_code.ts @@ -19,11 +19,13 @@ import { import { resetScopeManager, SCOPE_MANAGER } from "./scope.ts"; import * as scopeMethods from "./scope.ts"; import { resetTokens } from "./tokens.ts"; +import { tokens, tokensAndComments, initTokens, initTokensAndComments } from "./tokens.ts"; import * as tokenMethods from "./tokens.ts"; import { debugAssertIsNonNull } from "../utils/asserts.ts"; import type { Program } from "../generated/types.d.ts"; import type { Ranged } from "./location.ts"; +import type { Token, CommentToken } from "./tokens.ts"; import type { BufferWithArrays, Node } from "./types.ts"; import type { ScopeManager } from "./scope.ts"; @@ -178,6 +180,22 @@ export const SOURCE_CODE = Object.freeze({ return lines; }, + /** + * Array of all tokens and comments in the file, in source order. + */ + // This property is present in ESLint's `SourceCode`, but is undocumented + get tokensAndComments(): (Token | CommentToken)[] { + if (tokensAndComments === null) { + if (tokens === null) { + if (sourceText === null) initSourceText(); + initTokens(); + } + initTokensAndComments(); + } + debugAssertIsNonNull(tokensAndComments); + return tokensAndComments; + }, + /** * Get the source code for the given node. * @param node? - The AST node to get the text for. diff --git a/apps/oxlint/src-js/plugins/tokens.ts b/apps/oxlint/src-js/plugins/tokens.ts index feba86b072f29..c7ca66f11378a 100644 --- a/apps/oxlint/src-js/plugins/tokens.ts +++ b/apps/oxlint/src-js/plugins/tokens.ts @@ -135,7 +135,7 @@ const INCLUDE_COMMENTS_SKIP_OPTIONS: SkipOptions = { includeComments: true, skip // Created lazily only when needed. export let tokens: Token[] | null = null; let comments: CommentToken[] | null = null; -let tokensAndComments: (Token | CommentToken)[] | null = null; +export let tokensAndComments: (Token | CommentToken)[] | null = null; // TS-ESLint `parse` method. // Lazy-loaded only when needed, as it's a lot of code. @@ -178,7 +178,7 @@ export function initTokens() { * Caller must ensure `tokens` and `comments` are initialized before calling this function, * by calling `initTokens()` if `tokens === null`. */ -function initTokensAndComments() { +export function initTokensAndComments() { debugAssertIsNonNull(tokens); debugAssertIsNonNull(comments); diff --git a/apps/oxlint/test/fixtures/tokens/output.snap.md b/apps/oxlint/test/fixtures/tokens/output.snap.md index dd43a8ea50182..a8135c87ab077 100644 --- a/apps/oxlint/test/fixtures/tokens/output.snap.md +++ b/apps/oxlint/test/fixtures/tokens/output.snap.md @@ -25,7 +25,33 @@ 8 | `-> // Trailing comment `---- -Found 0 warnings and 1 error. + x tokens-plugin(tokens): Tokens and comments: + | Line loc=1:0-1:18 range=0-18 " Leading comment" + | Keyword loc=3:0-3:3 range=20-23 "let" + | Identifier loc=3:4-3:5 range=24-25 "x" + | Punctuator loc=3:6-3:7 range=26-27 "=" + | Block loc=3:8-3:28 range=28-48 " inline comment " + | Numeric loc=3:29-3:30 range=49-50 "1" + | Punctuator loc=3:30-3:31 range=50-51 ";" + | Line loc=5:0-5:18 range=53-71 " Another comment" + | Keyword loc=6:0-6:3 range=72-75 "let" + | Identifier loc=6:4-6:5 range=76-77 "y" + | Punctuator loc=6:6-6:7 range=78-79 "=" + | Numeric loc=6:8-6:9 range=80-81 "2" + | Punctuator loc=6:9-6:10 range=81-82 ";" + | Line loc=8:0-8:19 range=84-103 " Trailing comment" + ,-[files/index.js:1:1] + 1 | ,-> // Leading comment + 2 | | + 3 | | let x = /* inline comment */ 1; + 4 | | + 5 | | // Another comment + 6 | | let y = 2; + 7 | | + 8 | `-> // Trailing comment + `---- + +Found 0 warnings and 2 errors. Finished in Xms on 1 file using X threads. ``` diff --git a/apps/oxlint/test/fixtures/tokens/plugin.ts b/apps/oxlint/test/fixtures/tokens/plugin.ts index f7cbf9ed74449..92e8fef35b4b5 100644 --- a/apps/oxlint/test/fixtures/tokens/plugin.ts +++ b/apps/oxlint/test/fixtures/tokens/plugin.ts @@ -2,10 +2,14 @@ import type { Plugin, Rule } from "#oxlint"; const rule: Rule = { create(context) { - const { sourceCode } = context, - { ast } = sourceCode; + const { sourceCode } = context; - // Note: Comments should not appear in `ast.tokens` + // Get this first to check it works before `sourceText` or `ast` are accessed + const { tokensAndComments } = sourceCode; + + const { ast } = sourceCode; + + // `ast.tokens` does not include comments context.report({ message: `Tokens:\n` + @@ -23,6 +27,24 @@ const rule: Rule = { node: { range: [0, sourceCode.text.length] }, }); + // `sourceCode.tokensAndComments` does include comments + context.report({ + message: + `Tokens and comments:\n` + + tokensAndComments + .map( + ({ type, loc, range, value }) => + `${type.padEnd(17)} ` + + `loc=${loc.start.line}:${loc.start.column}-${loc.end.line}:${loc.end.column} `.padEnd( + 16, + ) + + `range=${range[0]}-${range[1]} `.padEnd(10) + + `"${value}"`, + ) + .join("\n"), + node: { range: [0, sourceCode.text.length] }, + }); + return {}; }, };