diff --git a/apps/oxlint/src-js/generated/constants.ts b/apps/oxlint/src-js/generated/constants.ts index 14f4f407d74af..35a5fc011caac 100644 --- a/apps/oxlint/src-js/generated/constants.ts +++ b/apps/oxlint/src-js/generated/constants.ts @@ -7,3 +7,6 @@ export const DATA_POINTER_POS_32 = 536870902; export const IS_TS_FLAG_POS = 2147483612; export const PROGRAM_OFFSET = 0; export const SOURCE_LEN_OFFSET = 16; +export const SOURCE_TYPE_OFFSET = 124; +export const SOURCE_TYPE_LANGUAGE_OFFSET = 0; +export const SOURCE_TYPE_VARIANT_OFFSET = 2; diff --git a/apps/oxlint/src-js/plugins/source_code.ts b/apps/oxlint/src-js/plugins/source_code.ts index bcc1019ad4130..0f27953edc6d2 100644 --- a/apps/oxlint/src-js/plugins/source_code.ts +++ b/apps/oxlint/src-js/plugins/source_code.ts @@ -1,4 +1,10 @@ -import { DATA_POINTER_POS_32, SOURCE_LEN_OFFSET } from "../generated/constants.ts"; +import { + DATA_POINTER_POS_32, + SOURCE_LEN_OFFSET, + SOURCE_TYPE_OFFSET, + SOURCE_TYPE_LANGUAGE_OFFSET, + SOURCE_TYPE_VARIANT_OFFSET, +} from "../generated/constants.ts"; // We use the deserializer which removes `ParenthesizedExpression`s from AST, // and with `range`, `loc`, and `parent` properties on AST nodes, to match ESLint @@ -66,6 +72,32 @@ export function initSourceText(): void { sourceText = textDecoder.decode(buffer.subarray(0, sourceByteLen)); } +/** + * Check if the current source is JSX/TSX. + * Reads the `SourceType.variant` (LanguageVariant) from the Program in the buffer. + * @returns `true` if the source is JSX or TSX + */ +export function isJsxSource(): boolean { + debugAssertIsNonNull(buffer); + const { uint32 } = buffer, + programPos = uint32[DATA_POINTER_POS_32], + sourceTypePos = programPos + SOURCE_TYPE_OFFSET; + return buffer[sourceTypePos + SOURCE_TYPE_VARIANT_OFFSET] === 1; +} + +/** + * Check if the current source is TypeScript. + * Reads the `SourceType.language` (Language) from the Program in the buffer. + * @returns `true` if the source is TypeScript or TypeScript Definition + */ +export function isTypescriptSource(): boolean { + debugAssertIsNonNull(buffer); + const { uint32 } = buffer, + programPos = uint32[DATA_POINTER_POS_32], + sourceTypePos = programPos + SOURCE_TYPE_OFFSET; + return buffer[sourceTypePos + SOURCE_TYPE_LANGUAGE_OFFSET] >= 1; +} + /** * Deserialize AST from buffer. */ diff --git a/apps/oxlint/src-js/plugins/tokens_parse.ts b/apps/oxlint/src-js/plugins/tokens_parse.ts index e21b63390074b..19914f7a41564 100644 --- a/apps/oxlint/src-js/plugins/tokens_parse.ts +++ b/apps/oxlint/src-js/plugins/tokens_parse.ts @@ -5,7 +5,7 @@ import { createRequire } from "node:module"; import { filePath } from "./context.ts"; import { getNodeLoc } from "./location.ts"; -import { sourceText } from "./source_code.ts"; +import { isJsxSource, isTypescriptSource, sourceText } from "./source_code.ts"; import { debugAssert, debugAssertIsNonNull } from "../utils/asserts.ts"; import type * as ts from "typescript"; @@ -51,6 +51,8 @@ export function parseTokens(): Token[] { tsSyntaxKind = tsModule.SyntaxKind; } + const scriptKind = getScriptKind(); + // Parse source text into TypeScript AST const tsAst = tsModule.createSourceFile( filePath, @@ -62,8 +64,7 @@ export function parseTokens(): Token[] { setExternalModuleIndicator: undefined, }, true, // `setParentNodes` - // TODO: Use `TS` or `TSX` depending on source type - tsModule.ScriptKind.TSX, + scriptKind, ); // Check that TypeScript hasn't altered source text. @@ -267,3 +268,20 @@ function hasJSXAncestor(node: ts.Node | undefined): boolean { function isJSXTokenKind(kind: ts.SyntaxKind): boolean { return kind >= tsSyntaxKind.JsxElement && kind <= tsSyntaxKind.JsxAttribute; } + +/** + * Determine TypeScript ScriptKind based on source type from the buffer. + * Reads the JSX flag from `SourceType.variant` and TypeScript flag from metadata. + * + * @returns Appropriate ScriptKind for the file + */ +function getScriptKind(): ts.ScriptKind { + debugAssertIsNonNull(tsModule); + const isJsx = isJsxSource(); + const isTs = isTypescriptSource(); + + if (isTs) { + return isJsx ? tsModule.ScriptKind.TSX : tsModule.ScriptKind.TS; + } + return isJsx ? tsModule.ScriptKind.JSX : tsModule.ScriptKind.JS; +} diff --git a/apps/oxlint/test/fixtures/tokens/files/generic_arrow.ts b/apps/oxlint/test/fixtures/tokens/files/generic_arrow.ts new file mode 100644 index 0000000000000..21b413c1a2ea4 --- /dev/null +++ b/apps/oxlint/test/fixtures/tokens/files/generic_arrow.ts @@ -0,0 +1,8 @@ +const obj = { + fn: (arg: T): T => { + return arg; + }, +}; + +// A comment after the object +export { obj }; diff --git a/apps/oxlint/test/fixtures/tokens/output.snap.md b/apps/oxlint/test/fixtures/tokens/output.snap.md index f0c84825581cb..811bee9a40b69 100644 --- a/apps/oxlint/test/fixtures/tokens/output.snap.md +++ b/apps/oxlint/test/fixtures/tokens/output.snap.md @@ -3,6 +3,330 @@ # stdout ``` + x tokens-plugin(tokens): Keyword ("const") + ,-[files/generic_arrow.ts:1:1] + 1 | const obj = { + : ^^^^^ + 2 | fn: (arg: T): T => { + `---- + + x tokens-plugin(tokens): Tokens: + | Keyword loc= 1:0 - 1:5 range= 0-5 "const" + | Identifier loc= 1:6 - 1:9 range= 6-9 "obj" + | Punctuator loc= 1:10 - 1:11 range= 10-11 "=" + | Punctuator loc= 1:12 - 1:13 range= 12-13 "{" + | Identifier loc= 2:2 - 2:4 range= 16-18 "fn" + | Punctuator loc= 2:4 - 2:5 range= 18-19 ":" + | Punctuator loc= 2:6 - 2:7 range= 20-21 "<" + | Identifier loc= 2:7 - 2:8 range= 21-22 "T" + | Punctuator loc= 2:8 - 2:9 range= 22-23 ">" + | Punctuator loc= 2:9 - 2:10 range= 23-24 "(" + | Identifier loc= 2:10 - 2:13 range= 24-27 "arg" + | Punctuator loc= 2:13 - 2:14 range= 27-28 ":" + | Identifier loc= 2:15 - 2:16 range= 29-30 "T" + | Punctuator loc= 2:16 - 2:17 range= 30-31 ")" + | Punctuator loc= 2:17 - 2:18 range= 31-32 ":" + | Identifier loc= 2:19 - 2:20 range= 33-34 "T" + | Punctuator loc= 2:21 - 2:23 range= 35-37 "=>" + | Punctuator loc= 2:24 - 2:25 range= 38-39 "{" + | Keyword loc= 3:4 - 3:10 range= 44-50 "return" + | Identifier loc= 3:11 - 3:14 range= 51-54 "arg" + | Punctuator loc= 3:14 - 3:15 range= 54-55 ";" + | Punctuator loc= 4:2 - 4:3 range= 58-59 "}" + | Punctuator loc= 4:3 - 4:4 range= 59-60 "," + | Punctuator loc= 5:0 - 5:1 range= 61-62 "}" + | Punctuator loc= 5:1 - 5:2 range= 62-63 ";" + | Keyword loc= 8:0 - 8:6 range= 95-101 "export" + | Punctuator loc= 8:7 - 8:8 range= 102-103 "{" + | Identifier loc= 8:9 - 8:12 range= 104-107 "obj" + | Punctuator loc= 8:13 - 8:14 range= 108-109 "}" + | Punctuator loc= 8:14 - 8:15 range= 109-110 ";" + ,-[files/generic_arrow.ts:1:1] + 1 | ,-> const obj = { + 2 | | fn: (arg: T): T => { + 3 | | return arg; + 4 | | }, + 5 | | }; + 6 | | + 7 | | // A comment after the object + 8 | `-> export { obj }; + `---- + + x tokens-plugin(tokens): Tokens and comments: + | Keyword loc= 1:0 - 1:5 range= 0-5 "const" + | Identifier loc= 1:6 - 1:9 range= 6-9 "obj" + | Punctuator loc= 1:10 - 1:11 range= 10-11 "=" + | Punctuator loc= 1:12 - 1:13 range= 12-13 "{" + | Identifier loc= 2:2 - 2:4 range= 16-18 "fn" + | Punctuator loc= 2:4 - 2:5 range= 18-19 ":" + | Punctuator loc= 2:6 - 2:7 range= 20-21 "<" + | Identifier loc= 2:7 - 2:8 range= 21-22 "T" + | Punctuator loc= 2:8 - 2:9 range= 22-23 ">" + | Punctuator loc= 2:9 - 2:10 range= 23-24 "(" + | Identifier loc= 2:10 - 2:13 range= 24-27 "arg" + | Punctuator loc= 2:13 - 2:14 range= 27-28 ":" + | Identifier loc= 2:15 - 2:16 range= 29-30 "T" + | Punctuator loc= 2:16 - 2:17 range= 30-31 ")" + | Punctuator loc= 2:17 - 2:18 range= 31-32 ":" + | Identifier loc= 2:19 - 2:20 range= 33-34 "T" + | Punctuator loc= 2:21 - 2:23 range= 35-37 "=>" + | Punctuator loc= 2:24 - 2:25 range= 38-39 "{" + | Keyword loc= 3:4 - 3:10 range= 44-50 "return" + | Identifier loc= 3:11 - 3:14 range= 51-54 "arg" + | Punctuator loc= 3:14 - 3:15 range= 54-55 ";" + | Punctuator loc= 4:2 - 4:3 range= 58-59 "}" + | Punctuator loc= 4:3 - 4:4 range= 59-60 "," + | Punctuator loc= 5:0 - 5:1 range= 61-62 "}" + | Punctuator loc= 5:1 - 5:2 range= 62-63 ";" + | Line loc= 7:0 - 7:29 range= 65-94 " A comment after the object" + | Keyword loc= 8:0 - 8:6 range= 95-101 "export" + | Punctuator loc= 8:7 - 8:8 range= 102-103 "{" + | Identifier loc= 8:9 - 8:12 range= 104-107 "obj" + | Punctuator loc= 8:13 - 8:14 range= 108-109 "}" + | Punctuator loc= 8:14 - 8:15 range= 109-110 ";" + ,-[files/generic_arrow.ts:1:1] + 1 | ,-> const obj = { + 2 | | fn: (arg: T): T => { + 3 | | return arg; + 4 | | }, + 5 | | }; + 6 | | + 7 | | // A comment after the object + 8 | `-> export { obj }; + `---- + + x tokens-plugin(tokens): Identifier ("obj") + ,-[files/generic_arrow.ts:1:7] + 1 | const obj = { + : ^^^ + 2 | fn: (arg: T): T => { + `---- + + x tokens-plugin(tokens): Punctuator ("=") + ,-[files/generic_arrow.ts:1:11] + 1 | const obj = { + : ^ + 2 | fn: (arg: T): T => { + `---- + + x tokens-plugin(tokens): Punctuator ("{") + ,-[files/generic_arrow.ts:1:13] + 1 | const obj = { + : ^ + 2 | fn: (arg: T): T => { + `---- + + x tokens-plugin(tokens): Identifier ("fn") + ,-[files/generic_arrow.ts:2:3] + 1 | const obj = { + 2 | fn: (arg: T): T => { + : ^^ + 3 | return arg; + `---- + + x tokens-plugin(tokens): Punctuator (":") + ,-[files/generic_arrow.ts:2:5] + 1 | const obj = { + 2 | fn: (arg: T): T => { + : ^ + 3 | return arg; + `---- + + x tokens-plugin(tokens): Punctuator ("<") + ,-[files/generic_arrow.ts:2:7] + 1 | const obj = { + 2 | fn: (arg: T): T => { + : ^ + 3 | return arg; + `---- + + x tokens-plugin(tokens): Identifier ("T") + ,-[files/generic_arrow.ts:2:8] + 1 | const obj = { + 2 | fn: (arg: T): T => { + : ^ + 3 | return arg; + `---- + + x tokens-plugin(tokens): Punctuator (">") + ,-[files/generic_arrow.ts:2:9] + 1 | const obj = { + 2 | fn: (arg: T): T => { + : ^ + 3 | return arg; + `---- + + x tokens-plugin(tokens): Punctuator ("(") + ,-[files/generic_arrow.ts:2:10] + 1 | const obj = { + 2 | fn: (arg: T): T => { + : ^ + 3 | return arg; + `---- + + x tokens-plugin(tokens): Identifier ("arg") + ,-[files/generic_arrow.ts:2:11] + 1 | const obj = { + 2 | fn: (arg: T): T => { + : ^^^ + 3 | return arg; + `---- + + x tokens-plugin(tokens): Punctuator (":") + ,-[files/generic_arrow.ts:2:14] + 1 | const obj = { + 2 | fn: (arg: T): T => { + : ^ + 3 | return arg; + `---- + + x tokens-plugin(tokens): Identifier ("T") + ,-[files/generic_arrow.ts:2:16] + 1 | const obj = { + 2 | fn: (arg: T): T => { + : ^ + 3 | return arg; + `---- + + x tokens-plugin(tokens): Punctuator (")") + ,-[files/generic_arrow.ts:2:17] + 1 | const obj = { + 2 | fn: (arg: T): T => { + : ^ + 3 | return arg; + `---- + + x tokens-plugin(tokens): Punctuator (":") + ,-[files/generic_arrow.ts:2:18] + 1 | const obj = { + 2 | fn: (arg: T): T => { + : ^ + 3 | return arg; + `---- + + x tokens-plugin(tokens): Identifier ("T") + ,-[files/generic_arrow.ts:2:20] + 1 | const obj = { + 2 | fn: (arg: T): T => { + : ^ + 3 | return arg; + `---- + + x tokens-plugin(tokens): Punctuator ("=>") + ,-[files/generic_arrow.ts:2:22] + 1 | const obj = { + 2 | fn: (arg: T): T => { + : ^^ + 3 | return arg; + `---- + + x tokens-plugin(tokens): Punctuator ("{") + ,-[files/generic_arrow.ts:2:25] + 1 | const obj = { + 2 | fn: (arg: T): T => { + : ^ + 3 | return arg; + `---- + + x tokens-plugin(tokens): Keyword ("return") + ,-[files/generic_arrow.ts:3:5] + 2 | fn: (arg: T): T => { + 3 | return arg; + : ^^^^^^ + 4 | }, + `---- + + x tokens-plugin(tokens): Identifier ("arg") + ,-[files/generic_arrow.ts:3:12] + 2 | fn: (arg: T): T => { + 3 | return arg; + : ^^^ + 4 | }, + `---- + + x tokens-plugin(tokens): Punctuator (";") + ,-[files/generic_arrow.ts:3:15] + 2 | fn: (arg: T): T => { + 3 | return arg; + : ^ + 4 | }, + `---- + + x tokens-plugin(tokens): Punctuator ("}") + ,-[files/generic_arrow.ts:4:3] + 3 | return arg; + 4 | }, + : ^ + 5 | }; + `---- + + x tokens-plugin(tokens): Punctuator (",") + ,-[files/generic_arrow.ts:4:4] + 3 | return arg; + 4 | }, + : ^ + 5 | }; + `---- + + x tokens-plugin(tokens): Punctuator ("}") + ,-[files/generic_arrow.ts:5:1] + 4 | }, + 5 | }; + : ^ + 6 | + `---- + + x tokens-plugin(tokens): Punctuator (";") + ,-[files/generic_arrow.ts:5:2] + 4 | }, + 5 | }; + : ^ + 6 | + `---- + + x tokens-plugin(tokens): Line (" A comment after the object") + ,-[files/generic_arrow.ts:7:1] + 6 | + 7 | // A comment after the object + : ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + 8 | export { obj }; + `---- + + x tokens-plugin(tokens): Keyword ("export") + ,-[files/generic_arrow.ts:8:1] + 7 | // A comment after the object + 8 | export { obj }; + : ^^^^^^ + `---- + + x tokens-plugin(tokens): Punctuator ("{") + ,-[files/generic_arrow.ts:8:8] + 7 | // A comment after the object + 8 | export { obj }; + : ^ + `---- + + x tokens-plugin(tokens): Identifier ("obj") + ,-[files/generic_arrow.ts:8:10] + 7 | // A comment after the object + 8 | export { obj }; + : ^^^ + `---- + + x tokens-plugin(tokens): Punctuator ("}") + ,-[files/generic_arrow.ts:8:14] + 7 | // A comment after the object + 8 | export { obj }; + : ^ + `---- + + x tokens-plugin(tokens): Punctuator (";") + ,-[files/generic_arrow.ts:8:15] + 7 | // A comment after the object + 8 | export { obj }; + : ^ + `---- + x tokens-plugin(tokens): Line (" Leading comment") ,-[files/index.js:1:1] 1 | // Leading comment @@ -161,8 +485,8 @@ : ^^^^^^^^^^^^^^^^^^^ `---- -Found 0 warnings and 16 errors. -Finished in Xms on 1 file using X threads. +Found 0 warnings and 49 errors. +Finished in Xms on 2 files using X threads. ``` # stderr diff --git a/napi/parser/src-js/generated/constants.js b/napi/parser/src-js/generated/constants.js index 14f4f407d74af..35a5fc011caac 100644 --- a/napi/parser/src-js/generated/constants.js +++ b/napi/parser/src-js/generated/constants.js @@ -7,3 +7,6 @@ export const DATA_POINTER_POS_32 = 536870902; export const IS_TS_FLAG_POS = 2147483612; export const PROGRAM_OFFSET = 0; export const SOURCE_LEN_OFFSET = 16; +export const SOURCE_TYPE_OFFSET = 124; +export const SOURCE_TYPE_LANGUAGE_OFFSET = 0; +export const SOURCE_TYPE_VARIANT_OFFSET = 2; diff --git a/tasks/ast_tools/src/generators/raw_transfer.rs b/tasks/ast_tools/src/generators/raw_transfer.rs index f2abaa49da7b2..06d47c82ec984 100644 --- a/tasks/ast_tools/src/generators/raw_transfer.rs +++ b/tasks/ast_tools/src/generators/raw_transfer.rs @@ -1271,6 +1271,12 @@ struct Constants { program_offset: u32, /// Offset of `u32` source text length, relative to position of `Program` source_len_offset: u32, + /// Offset of `SourceType` struct, relative to position of `Program` + source_type_offset: u32, + /// Offset of `Language` field within `SourceType` + source_type_language_offset: u32, + /// Offset of `LanguageVariant` field within `SourceType` + source_type_variant_offset: u32, /// Size of `RawTransferData` in bytes raw_metadata_size: u32, } @@ -1283,6 +1289,9 @@ fn generate_constants(consts: Constants) -> (String, TokenStream) { is_ts_pos, program_offset, source_len_offset, + source_type_offset, + source_type_language_offset, + source_type_variant_offset, raw_metadata_size, } = consts; @@ -1296,6 +1305,9 @@ fn generate_constants(consts: Constants) -> (String, TokenStream) { export const IS_TS_FLAG_POS = {is_ts_pos}; export const PROGRAM_OFFSET = {program_offset}; export const SOURCE_LEN_OFFSET = {source_len_offset}; + export const SOURCE_TYPE_OFFSET = {source_type_offset}; + export const SOURCE_TYPE_LANGUAGE_OFFSET = {source_type_language_offset}; + export const SOURCE_TYPE_VARIANT_OFFSET = {source_type_variant_offset}; "); let block_size = number_lit(BLOCK_SIZE); @@ -1362,13 +1374,15 @@ fn get_constants(schema: &Schema) -> Constants { .field_by_name("program") .offset_64(); - let source_len_offset = schema - .type_by_name("Program") - .as_struct() - .unwrap() - .field_by_name("source_text") - .offset_64() - + STR_LEN_OFFSET; + let program_struct = schema.type_by_name("Program").as_struct().unwrap(); + + let source_len_offset = + program_struct.field_by_name("source_text").offset_64() + STR_LEN_OFFSET; + + let source_type_offset = program_struct.field_by_name("source_type").offset_64(); + let source_type_struct = schema.type_by_name("SourceType").as_struct().unwrap(); + let source_type_language_offset = source_type_struct.field_by_name("language").offset_64(); + let source_type_variant_offset = source_type_struct.field_by_name("variant").offset_64(); Constants { buffer_size, @@ -1376,6 +1390,9 @@ fn get_constants(schema: &Schema) -> Constants { is_ts_pos, program_offset, source_len_offset, + source_type_offset, + source_type_language_offset, + source_type_variant_offset, raw_metadata_size, } }