Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/bundler/defines.rs
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,7 @@ impl DefineExt for Define {
identifiers: StringHashMap::default(),
dots: StringHashMap::default(),
drop_debugger,
user_hash: None,
});
define.dots.reserve(124);

Expand Down
30 changes: 23 additions & 7 deletions src/bundler/options.rs
Original file line number Diff line number Diff line change
Expand Up @@ -946,12 +946,31 @@ pub(crate) fn defines_from_transform_options(

let drop_debugger = drop.iter().any(|item| *item == b"debugger");

Ok(defines::Define::init(
let user_hash = defines::Define::hash_user_inputs(
user_defines
.keys()
.iter()
.zip(user_defines.values().iter())
.map(|(k, v)| (k.as_ref(), v.as_ref())),
environment_defines
.keys()
.iter()
.zip(environment_defines.values().iter())
.filter_map(|(k, v)| match &v.value {
defines::DefineValue::EString(s) if s.is_utf8() => Some((k.as_ref(), s.slice8())),
_ => None,
}),
drop.iter().copied(),
);

let mut define = defines::Define::init(
Some(resolved_defines),
Some(environment_defines),
drop_debugger,
omit_unused_global_calls,
)?)
)?;
define.user_hash = user_hash;
Ok(define)
}

const DEFAULT_LOADER_EXT_BUN: &[&[u8]] = &[b".node", b".html"];
Expand Down Expand Up @@ -1414,6 +1433,7 @@ impl<'a> BundleOptions<'a> {
identifiers: self.define.identifiers.clone(),
dots: self.define.dots.clone(),
drop_debugger: self.define.drop_debugger,
user_hash: self.define.user_hash,
}),
drop: self.drop.clone(),
bundler_feature_flags: self
Expand Down Expand Up @@ -1663,11 +1683,7 @@ impl<'a> BundleOptions<'a> {
log,
// `define` is filled by `load_defines` later;
// initialize empty so the struct is well-formed before `load_defines` runs.
define: Box::new(defines::Define {
identifiers: Default::default(),
dots: Default::default(),
drop_debugger: false,
}),
define: Box::new(defines::Define::default()),
loaders,
output_dir: Box::from(transform.output_dir.as_deref().unwrap_or(b"out")),
target,
Expand Down
1 change: 1 addition & 0 deletions src/bundler/transpiler.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1644,6 +1644,7 @@ impl<'a> Transpiler<'a> {
.bundler_feature_flags
.as_deref()
.and_then(|s| s.clone().ok().map(Box::new));
opts.features.define_hash = self.options.define.user_hash;
opts.features.repl_mode = self.options.repl_mode;

// we'll just always enable top-level await
Expand Down
39 changes: 39 additions & 0 deletions src/js_parser/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -493,9 +493,48 @@ pub mod defines {
pub identifiers: StringHashMap<IdentifierDefine>,
pub dots: StringHashMap<Vec<DotDefine>>,
pub drop_debugger: bool,
/// `hash_user_inputs` of this table's inputs, for the runtime transpiler cache key.
pub user_hash: Option<u64>,
}

impl Define {
/// Order-independent, length-prefixed hash of the three inputs. `None` when all are empty.
pub fn hash_user_inputs<'i>(
defines: impl IntoIterator<Item = (&'i [u8], &'i [u8])>,
env_defines: impl IntoIterator<Item = (&'i [u8], &'i [u8])>,
drop: impl IntoIterator<Item = &'i [u8]>,
) -> Option<u64> {
let mut defines: Vec<(&[u8], &[u8])> = defines.into_iter().collect();
let mut env_defines: Vec<(&[u8], &[u8])> = env_defines.into_iter().collect();
let mut drop: Vec<&[u8]> = drop.into_iter().filter(|item| !item.is_empty()).collect();
if defines.is_empty() && env_defines.is_empty() && drop.is_empty() {
return None;
}
defines.sort_unstable();
env_defines.sort_unstable();
drop.sort_unstable();
drop.dedup();

let mut hasher = bun_wyhash::Wyhash::init(0);
let mut update = |bytes: &[u8]| {
hasher.update(&(bytes.len() as u64).to_le_bytes());
hasher.update(bytes);
};
// Separate sections: `init` lets a later env pair override a user pair.
for pairs in [defines, env_defines] {
update(&(pairs.len() as u64).to_le_bytes());
for (key, value) in pairs {
update(key);
update(value);
}
}
update(&(drop.len() as u64).to_le_bytes());
for item in drop {
update(item);
}
Some(hasher.final_())
}

pub(crate) fn for_identifier(&self, name: &[u8]) -> Option<&IdentifierDefine> {
if let Some(data) = self.identifiers.get(name) {
return Some(data);
Expand Down
1 change: 1 addition & 0 deletions src/js_parser/parse/parse_entry.rs
Original file line number Diff line number Diff line change
Expand Up @@ -212,6 +212,7 @@ impl<'a> Options<'a> {
runtime_transpiler_cache: None,
lower_using: f.lower_using,
bundler_feature_flags: None,
define_hash: f.define_hash,
repl_mode: f.repl_mode,
jsx_optimization_inline: f.jsx_optimization_inline,
},
Expand Down
10 changes: 10 additions & 0 deletions src/js_parser/parser.rs
Original file line number Diff line number Diff line change
Expand Up @@ -257,6 +257,9 @@ pub mod Runtime {
/// in watch/dev-server mode.
pub bundler_feature_flags: Option<Box<StringSet>>,

/// `Define::user_hash` of this parse's define table (runtime transpiler cache key).
pub define_hash: Option<u64>,

/// REPL mode: transforms code for interactive evaluation
/// - Wraps lone object literals `{...}` in parentheses
/// - Hoists variable declarations for REPL persistence
Expand Down Expand Up @@ -304,6 +307,7 @@ pub mod Runtime {
runtime_transpiler_cache: None,
lower_using: true,
bundler_feature_flags: None,
define_hash: None,
repl_mode: false,
jsx_optimization_inline: false,
}
Expand Down Expand Up @@ -391,6 +395,12 @@ pub mod Runtime {
hasher.update(b"\x00");
}
}

// Define pairs and `--drop` entries. `None` adds nothing, like an empty flag set.
if let Some(define_hash) = self.define_hash {
hasher.update(b"define");
hasher.update(&define_hash.to_le_bytes());
}
}

pub(crate) fn should_unwrap_require(&self, package_name: &[u8]) -> bool {
Expand Down
3 changes: 2 additions & 1 deletion src/jsc/RuntimeTranspilerCache.rs
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,8 @@ bun_core::declare_scope!(cache, visible);
/// offsets picked by a header byte) plus a body of tagged records with
/// u8/u16/u32 ids and implied slots dropped, instead of fixed u32 arrays.
/// Version 27: ModuleInfo string table holds Latin-1 / UTF-16 bodies, not WTF-8.
const EXPECTED_VERSION: u32 = 27;
/// Version 28: the define table and `--drop` entries participate in the features hash.
const EXPECTED_VERSION: u32 = 28;

/// Source files smaller than this are not written to / read from the on-disk
/// transpiler cache. Originally 50 KiB, which excluded almost every file in a
Expand Down
7 changes: 0 additions & 7 deletions src/runtime/cli/Arguments.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1569,13 +1569,6 @@ pub(crate) fn parse(cmd: CommandTag, ctx: Context<'_>) -> crate::Result<api::Tra
};
}
}

if let Some(define) = &opts.define {
if !define.keys.is_empty() {
bun_jsc::runtime_transpiler_cache::IS_DISABLED
.store(true, std::sync::atomic::Ordering::Relaxed);
}
}
}

if matches!(
Expand Down
133 changes: 133 additions & 0 deletions test/cli/run/transpiler-cache.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -320,6 +320,139 @@ describe("transpiler cache", () => {
expect(newCacheCount()).toBe(0); // cache hit, order doesn't matter
});

// A define replaces an identifier at parse time, so the define table is part
// of the cache key. The cache is shared by every project of the user, and
// keyed by source bytes, so two projects with the same file must not see
// each other's values.
describe("defines are part of the cache key", () => {
const code = `console.log(typeof BVAL === "undefined" ? "undefined" : BVAL);`;
const filler = Buffer.alloc((50 * 1024 * 1.5) | 0, "/").toString();

const run = (cwd: string, args: string[]) => {
const result = Bun.spawnSync({ cmd: [bunExe(), ...args], cwd, env });
if (!result.success) throw new Error(result.stderr.toString());
return result.stdout.toString().trim();
};

test("bunfig [define] invalidates cache", () => {
// `bun run <file>` loads bunfig.toml after the command line is parsed,
// so the define table only exists once the runtime is up.
const projectA = join(temp_dir, "a");
const projectB = join(temp_dir, "b");
for (const dir of [projectA, projectB]) {
mkdirSync(dir);
writeFileSync(join(dir, "a.js"), code + "\n//" + filler);
}
const setDefine = (dir: string, value: string | null) => {
if (value === null) rmSync(join(dir, "bunfig.toml"), { force: true });
else writeFileSync(join(dir, "bunfig.toml"), `[define]\nBVAL = '${JSON.stringify(value)}'\n`);
};

setDefine(projectA, "one");
expect(run(projectA, ["run", "./a.js"])).toBe("one");
expect(newCacheCount()).toBe(1);
expect(run(projectA, ["run", "./a.js"])).toBe("one");
expect(newCacheCount()).toBe(0);

// A new value: features_hash differs -> old entry deleted, new entry written
setDefine(projectA, "two");
expect(run(projectA, ["run", "./a.js"])).toBe("two");
expect(newCacheCount()).toBe(0);

setDefine(projectA, null);
expect(run(projectA, ["run", "./a.js"])).toBe("undefined");
expect(newCacheCount()).toBe(0);

// The same source bytes in another project, with its own define
setDefine(projectB, "mine");
expect(run(projectB, ["run", "./a.js"])).toBe("mine");
expect(newCacheCount()).toBe(0);
expect(run(projectB, ["./a.js"])).toBe("mine");
expect(newCacheCount()).toBe(0);
});

test("--define invalidates cache", () => {
writeFileSync(join(temp_dir, "a.js"), code + "\n//" + filler);

expect(run(temp_dir, ["--define", 'BVAL:"cli"', "a.js"])).toBe("cli");
expect(existsSync(cache_dir)).toBeTrue();
expect(newCacheCount()).toBe(1);
expect(run(temp_dir, ["--define", 'BVAL:"cli"', "a.js"])).toBe("cli");
expect(newCacheCount()).toBe(0);

expect(run(temp_dir, ["--define", 'BVAL:"other"', "a.js"])).toBe("other");
expect(newCacheCount()).toBe(0);

expect(run(temp_dir, ["a.js"])).toBe("undefined");
expect(newCacheCount()).toBe(0);

expect(run(temp_dir, ["run", "--define", 'BVAL:"cli"', "./a.js"])).toBe("cli");
expect(newCacheCount()).toBe(0);

// The key is built from the resolved map: flag order does not matter,
// and a key given twice keeps its last value, so `x` then `cli` is
// served the entry that `--define BVAL:"cli"` alone wrote above.
expect(run(temp_dir, ["--define", 'BVAL:"cli"', "--define", "OTHER:1", "a.js"])).toBe("cli");
expect(newCacheCount()).toBe(0);
const entry = join(cache_dir, readdirSync(cache_dir)[0]);
const written = readFileSync(entry);
expect(run(temp_dir, ["--define", "OTHER:1", "--define", 'BVAL:"cli"', "a.js"])).toBe("cli");
expect(readFileSync(entry).equals(written)).toBeTrue();

expect(run(temp_dir, ["--define", 'BVAL:"cli"', "a.js"])).toBe("cli");
const alone = readFileSync(entry);
expect(alone.equals(written)).toBeFalse();
expect(run(temp_dir, ["--define", 'BVAL:"x"', "--define", 'BVAL:"cli"', "a.js"])).toBe("cli");
expect(readFileSync(entry).equals(alone)).toBeTrue();
expect(newCacheCount()).toBe(0);
});

test("--define passed to bun test invalidates cache", () => {
// `bun test` builds its define table on its own boot path. The test file
// is below the minimum cache size; the module it imports is not, and
// `bun a.js` loads that same module.
writeFileSync(join(temp_dir, "a.js"), code + "\n//" + filler);
writeFileSync(
join(temp_dir, "a.test.js"),
`import "./a.js";\nimport { test } from "bun:test";\ntest("x", () => {});\n`,
);
// `bun test` prints its version banner to stdout ahead of the module's output.
const lastLine = (args: string[]) => run(temp_dir, args).split("\n").at(-1);

expect(lastLine(["test", "--define", 'BVAL:"cli"', "./a.test.js"])).toBe("cli");
expect(newCacheCount()).toBe(1);

expect(lastLine(["test", "./a.test.js"])).toBe("undefined");
expect(newCacheCount()).toBe(0);
expect(run(temp_dir, ["a.js"])).toBe("undefined");
expect(newCacheCount()).toBe(0);

expect(lastLine(["test", "--define", 'BVAL:"cli"', "./a.test.js"])).toBe("cli");
expect(newCacheCount()).toBe(0);
expect(run(temp_dir, ["a.js"])).toBe("undefined");
expect(newCacheCount()).toBe(0);
});

test("--drop invalidates cache", () => {
writeFileSync(
join(temp_dir, "a.js"),
`console.log("logged");\nprocess.stdout.write("written\\n");` + "\n//" + filler,
);

expect(run(temp_dir, ["a.js"])).toBe("logged\nwritten");
expect(newCacheCount()).toBe(1);

expect(run(temp_dir, ["--drop=console", "a.js"])).toBe("written");
expect(newCacheCount()).toBe(0);

expect(run(temp_dir, ["--drop=console", "a.js"])).toBe("written");
expect(newCacheCount()).toBe(0);

expect(run(temp_dir, ["a.js"])).toBe("logged\nwritten");
expect(newCacheCount()).toBe(0);
});
});

// Serving the entry point from the cache must not change how the modules it
// loads are resolved. Both of these are gated on the `has_loaded` flag, which
// used to be set only on the path that runs the printer.
Expand Down
Loading