diff --git a/.github/generated/ast_changes_watch_list.yml b/.github/generated/ast_changes_watch_list.yml index 4c15e24edeaa2..2a3556935ad9c 100644 --- a/.github/generated/ast_changes_watch_list.yml +++ b/.github/generated/ast_changes_watch_list.yml @@ -60,6 +60,7 @@ src: - 'crates/oxc_syntax/src/serialize.rs' - 'crates/oxc_syntax/src/symbol.rs' - 'crates/oxc_traverse/src/generated/scopes_collector.rs' + - 'napi/parser/generated/constants.js' - 'napi/parser/generated/deserialize/js.js' - 'napi/parser/generated/deserialize/ts.js' - 'napi/parser/generated/lazy/constructors.js' diff --git a/napi/parser/bench.bench.mjs b/napi/parser/bench.bench.mjs index 8769c072abee3..02e645c6cc5cc 100644 --- a/napi/parser/bench.bench.mjs +++ b/napi/parser/bench.bench.mjs @@ -8,6 +8,7 @@ import { experimentalGetLazyVisitor, parseAsync, parseSync } from './index.js'; // Use `require` not `import` to load these internal modules, to avoid evaluating the modules // twice as ESM and CJS const require = createRequire(import.meta.filename); +const { DATA_POINTER_POS_32, PROGRAM_OFFSET } = require('./generated/constants.js'); const deserializeJS = require('./generated/deserialize/js.js'); const deserializeTS = require('./generated/deserialize/ts.js'); const { isJsAst, prepareRaw, returnBufferToCache } = require('./raw-transfer/common.js'); @@ -180,9 +181,7 @@ for (const { filename, code } of fixtures) { token: TOKEN, }; - // (2 * 1024 * 1024 * 1024 - 16) >> 2 - const metadataPos32 = 536870908; - const programPos = buffer.uint32[metadataPos32]; + const programPos = buffer.uint32[DATA_POINTER_POS_32] + PROGRAM_OFFSET; benchRaw('parser_napi_raw_lazy_visit_only(debugger)', () => { ast.nodes = new Map(); diff --git a/napi/parser/generated/constants.js b/napi/parser/generated/constants.js new file mode 100644 index 0000000000000..a83df534fa01b --- /dev/null +++ b/napi/parser/generated/constants.js @@ -0,0 +1,8 @@ +// Auto-generated code, DO NOT EDIT DIRECTLY! +// To edit this generated file you have to edit `tasks/ast_tools/src/generators/raw_transfer.rs`. + +const DATA_POINTER_POS_32 = 536870908, + IS_TS_FLAG_POS = 2147483636, + PROGRAM_OFFSET = 0; + +module.exports = { DATA_POINTER_POS_32, IS_TS_FLAG_POS, PROGRAM_OFFSET }; diff --git a/napi/parser/generated/deserialize/js.js b/napi/parser/generated/deserialize/js.js index da3e93fd93bf3..3336bfa449517 100644 --- a/napi/parser/generated/deserialize/js.js +++ b/napi/parser/generated/deserialize/js.js @@ -20,10 +20,7 @@ function deserialize(buffer, sourceTextInput, sourceLenInput) { sourceLen = sourceLenInput; sourceIsAscii = sourceText.length === sourceLen; - // (2 * 1024 * 1024 * 1024 - 16) >> 2 - const metadataPos32 = 536870908; - - const data = deserializeRawTransferData(uint32[metadataPos32]); + const data = deserializeRawTransferData(uint32[536870908]); uint8 = uint32 = diff --git a/napi/parser/generated/deserialize/ts.js b/napi/parser/generated/deserialize/ts.js index f7006757c22a9..e8ed3a0a0294a 100644 --- a/napi/parser/generated/deserialize/ts.js +++ b/napi/parser/generated/deserialize/ts.js @@ -20,10 +20,7 @@ function deserialize(buffer, sourceTextInput, sourceLenInput) { sourceLen = sourceLenInput; sourceIsAscii = sourceText.length === sourceLen; - // (2 * 1024 * 1024 * 1024 - 16) >> 2 - const metadataPos32 = 536870908; - - const data = deserializeRawTransferData(uint32[metadataPos32]); + const data = deserializeRawTransferData(uint32[536870908]); uint8 = uint32 = diff --git a/napi/parser/package.json b/napi/parser/package.json index e0cbf0fcd59f3..20c80fffc3330 100644 --- a/napi/parser/package.json +++ b/napi/parser/package.json @@ -44,6 +44,7 @@ "wasm.mjs", "bindings.js", "webcontainer-fallback.js", + "generated/constants.js", "generated/deserialize/js.js", "generated/deserialize/ts.js", "generated/lazy/constructors.js", diff --git a/napi/parser/raw-transfer/common.js b/napi/parser/raw-transfer/common.js index 8832002e89f33..8f9c57869b1bb 100644 --- a/napi/parser/raw-transfer/common.js +++ b/napi/parser/raw-transfer/common.js @@ -7,6 +7,7 @@ const { parseAsyncRaw: parseAsyncRawBinding, getBufferOffset, } = require('../bindings.js'); +const { IS_TS_FLAG_POS } = require('../generated/constants.js'); module.exports = { parseSyncRawImpl, @@ -227,9 +228,7 @@ function prepareRaw(sourceText) { * @returns {boolean} - `true` if AST is JS, `false` if TS */ function isJsAst(buffer) { - // 2147483636 = (2 * 1024 * 1024 * 1024) - 12 - // i.e. 12 bytes from end of 2 GiB buffer - return buffer[2147483636] === 0; + return buffer[IS_TS_FLAG_POS] === 0; } /** diff --git a/napi/parser/raw-transfer/lazy.js b/napi/parser/raw-transfer/lazy.js index 28a2bb337cdad..9ea0cd961fc23 100644 --- a/napi/parser/raw-transfer/lazy.js +++ b/napi/parser/raw-transfer/lazy.js @@ -2,6 +2,7 @@ const { parseSyncRawImpl, parseAsyncRawImpl, returnBufferToCache } = require('./common.js'), { TOKEN } = require('./lazy-common.js'), + { DATA_POINTER_POS_32, PROGRAM_OFFSET } = require('../generated/constants.js'), { RawTransferData } = require('../generated/lazy/constructors.js'), walkProgram = require('../generated/lazy/walk.js'), { Visitor, getVisitorsArr } = require('./visitor.js'); @@ -102,9 +103,7 @@ function construct(buffer, sourceText, sourceLen) { bufferRecycleRegistry.register(ast, buffer, ast); // Get root data class instance - // (2 * 1024 * 1024 * 1024 - 16) >> 2 - const metadataPos32 = 536870908; - const rawDataPos = buffer.uint32[metadataPos32]; + const rawDataPos = buffer.uint32[DATA_POINTER_POS_32]; const data = new RawTransferData(rawDataPos, ast); return { @@ -122,7 +121,7 @@ function construct(buffer, sourceText, sourceLen) { }, dispose: dispose.bind(null, ast), visit(visitor) { - walkProgram(rawDataPos, ast, getVisitorsArr(visitor)); + walkProgram(rawDataPos + PROGRAM_OFFSET, ast, getVisitorsArr(visitor)); }, }; } diff --git a/tasks/ast_tools/src/generators/raw_transfer.rs b/tasks/ast_tools/src/generators/raw_transfer.rs index ce902a61fd8b4..5685bd7801253 100644 --- a/tasks/ast_tools/src/generators/raw_transfer.rs +++ b/tasks/ast_tools/src/generators/raw_transfer.rs @@ -24,6 +24,13 @@ use crate::{ use super::define_generator; +/// Size of raw transfer buffer +const BUFFER_SIZE: u32 = 1 << 31; // 2 GiB +/// Size of metadata written to end of buffer. +const METADATA_SIZE: u32 = 16; +/// Offset of flag for whether AST is TypeScript or not, relative to start of metadata. +const IS_TS_FLAG_OFFSET: u32 = 4; + // Offsets of `Vec`'s fields. // `Vec` is `#[repr(transparent)]` and `RawVec` is `#[repr(C)]`, so these offsets are fixed. pub(super) const VEC_PTR_FIELD_OFFSET: usize = 0; @@ -36,7 +43,11 @@ define_generator!(RawTransferGenerator); impl Generator for RawTransferGenerator { fn generate_many(&self, schema: &Schema, codegen: &Codegen) -> Vec { - let Codes { js, ts, .. } = generate_deserializers(schema, codegen); + let consts = get_constants(schema); + + let Codes { js, ts, .. } = generate_deserializers(consts, schema, codegen); + let constants = generate_constants(consts); + vec![ Output::Javascript { path: format!("{NAPI_PARSER_PACKAGE_PATH}/generated/deserialize/js.js"), @@ -46,43 +57,14 @@ impl Generator for RawTransferGenerator { path: format!("{NAPI_PARSER_PACKAGE_PATH}/generated/deserialize/ts.js"), code: ts, }, + Output::Javascript { + path: format!("{NAPI_PARSER_PACKAGE_PATH}/generated/constants.js"), + code: constants, + }, ] } } -/// Prelude to generated deserializer. -/// Defines the main `deserialize` function. -static PRELUDE: &str = " - 'use strict'; - - module.exports = deserialize; - - let uint8, uint32, float64, sourceText, sourceIsAscii, sourceLen; - - const textDecoder = new TextDecoder('utf-8', { ignoreBOM: true }), - decodeStr = textDecoder.decode.bind(textDecoder), - { fromCodePoint } = String; - - function deserialize(buffer, sourceTextInput, sourceLenInput) { - uint8 = buffer; - uint32 = buffer.uint32; - float64 = buffer.float64; - - sourceText = sourceTextInput; - sourceLen = sourceLenInput; - sourceIsAscii = sourceText.length === sourceLen; - - // (2 * 1024 * 1024 * 1024 - 16) >> 2 - const metadataPos32 = 536870908; - - const data = deserializeRawTransferData(uint32[metadataPos32]); - - uint8 = uint32 = float64 = sourceText = undefined; - - return data; - } -"; - /// Container for generated code. struct Codes { /// Code which is part of JS deserializer only @@ -94,10 +76,43 @@ struct Codes { } /// Generate deserializer functions for all types. -fn generate_deserializers(schema: &Schema, codegen: &Codegen) -> Codes { +fn generate_deserializers(consts: Constants, schema: &Schema, codegen: &Codegen) -> Codes { let estree_derive_id = codegen.get_derive_id_by_name("ESTree"); - let mut codes = Codes { js: PRELUDE.to_string(), ts: PRELUDE.to_string(), both: String::new() }; + // Prelude to generated deserializer. + // Defines the main `deserialize` function. + let Constants { data_pointer_pos_32, .. } = consts; + + #[rustfmt::skip] + let prelude = format!(" + 'use strict'; + + module.exports = deserialize; + + let uint8, uint32, float64, sourceText, sourceIsAscii, sourceLen; + + const textDecoder = new TextDecoder('utf-8', {{ ignoreBOM: true }}), + decodeStr = textDecoder.decode.bind(textDecoder), + {{ fromCodePoint }} = String; + + function deserialize(buffer, sourceTextInput, sourceLenInput) {{ + uint8 = buffer; + uint32 = buffer.uint32; + float64 = buffer.float64; + + sourceText = sourceTextInput; + sourceLen = sourceLenInput; + sourceIsAscii = sourceText.length === sourceLen; + + const data = deserializeRawTransferData(uint32[{data_pointer_pos_32}]); + + uint8 = uint32 = float64 = sourceText = undefined; + + return data; + }} + "); + + let mut codes = Codes { js: prelude.clone(), ts: prelude, both: String::new() }; for type_def in &schema.types { match type_def { @@ -946,3 +961,42 @@ impl DeserializeFunctionName for CellDef { self.inner_type(schema).plain_name(schema) } } + +/// Constants for position of fields in buffer which deserialization starts from. +#[derive(Clone, Copy)] +struct Constants { + data_pointer_pos_32: u32, + is_ts_pos: u32, + program_offset: u32, +} + +/// Generate constants file. +fn generate_constants(consts: Constants) -> String { + let Constants { data_pointer_pos_32, is_ts_pos, program_offset } = consts; + + #[rustfmt::skip] + let output = format!(" + const DATA_POINTER_POS_32 = {data_pointer_pos_32}, + IS_TS_FLAG_POS = {is_ts_pos}, + PROGRAM_OFFSET = {program_offset}; + + module.exports = {{ DATA_POINTER_POS_32, IS_TS_FLAG_POS, PROGRAM_OFFSET }}; + "); + output +} + +/// Calculate constants. +fn get_constants(schema: &Schema) -> Constants { + let metadata_pos = BUFFER_SIZE - METADATA_SIZE; + let data_pointer_pos_32 = metadata_pos / 4; + let is_ts_pos = metadata_pos + IS_TS_FLAG_OFFSET; + + let program_offset = schema + .type_by_name("RawTransferData") + .as_struct() + .unwrap() + .field_by_name("program") + .offset_64(); + + Constants { data_pointer_pos_32, is_ts_pos, program_offset } +}