Skip to content
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/bundler/defines.rs
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,7 @@ impl DefineExt for Define {
identifiers: StringHashMap::default(),
dots: StringHashMap::default(),
drop_debugger,
user_hash: None,
});
define.dots.reserve(124);

Expand Down
30 changes: 23 additions & 7 deletions src/bundler/options.rs
Original file line number Diff line number Diff line change
Expand Up @@ -946,12 +946,31 @@ pub(crate) fn defines_from_transform_options(

let drop_debugger = drop.iter().any(|item| *item == b"debugger");

Ok(defines::Define::init(
let user_hash = defines::Define::hash_user_inputs(
user_defines
.keys()
.iter()
.zip(user_defines.values().iter())
.map(|(k, v)| (k.as_ref(), v.as_ref())),
environment_defines
.keys()
.iter()
.zip(environment_defines.values().iter())
.filter_map(|(k, v)| match &v.value {
defines::DefineValue::EString(s) if s.is_utf8() => Some((k.as_ref(), s.slice8())),
_ => None,
}),
drop.iter().copied(),
);

let mut define = defines::Define::init(
Some(resolved_defines),
Some(environment_defines),
drop_debugger,
omit_unused_global_calls,
)?)
)?;
define.user_hash = user_hash;
Ok(define)
}

const DEFAULT_LOADER_EXT_BUN: &[&[u8]] = &[b".node", b".html"];
Expand Down Expand Up @@ -1414,6 +1433,7 @@ impl<'a> BundleOptions<'a> {
identifiers: self.define.identifiers.clone(),
dots: self.define.dots.clone(),
drop_debugger: self.define.drop_debugger,
user_hash: self.define.user_hash,
}),
drop: self.drop.clone(),
bundler_feature_flags: self
Expand Down Expand Up @@ -1663,11 +1683,7 @@ impl<'a> BundleOptions<'a> {
log,
// `define` is filled by `load_defines` later;
// initialize empty so the struct is well-formed before `load_defines` runs.
define: Box::new(defines::Define {
identifiers: Default::default(),
dots: Default::default(),
drop_debugger: false,
}),
define: Box::new(defines::Define::default()),
loaders,
output_dir: Box::from(transform.output_dir.as_deref().unwrap_or(b"out")),
target,
Expand Down
1 change: 1 addition & 0 deletions src/bundler/transpiler.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1644,6 +1644,7 @@ impl<'a> Transpiler<'a> {
.bundler_feature_flags
.as_deref()
.and_then(|s| s.clone().ok().map(Box::new));
opts.features.define_hash = self.options.define.user_hash;
opts.features.repl_mode = self.options.repl_mode;

// we'll just always enable top-level await
Expand Down
39 changes: 39 additions & 0 deletions src/js_parser/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -493,9 +493,48 @@ pub mod defines {
pub identifiers: StringHashMap<IdentifierDefine>,
pub dots: StringHashMap<Vec<DotDefine>>,
pub drop_debugger: bool,
/// `hash_user_inputs` of this table's inputs, for the runtime transpiler cache key.
pub user_hash: Option<u64>,
}

impl Define {
/// Order-independent, length-prefixed hash of the three inputs. `None` when all are empty.
pub fn hash_user_inputs<'i>(
defines: impl IntoIterator<Item = (&'i [u8], &'i [u8])>,
env_defines: impl IntoIterator<Item = (&'i [u8], &'i [u8])>,
drop: impl IntoIterator<Item = &'i [u8]>,
) -> Option<u64> {
let mut defines: Vec<(&[u8], &[u8])> = defines.into_iter().collect();
let mut env_defines: Vec<(&[u8], &[u8])> = env_defines.into_iter().collect();
let mut drop: Vec<&[u8]> = drop.into_iter().filter(|item| !item.is_empty()).collect();
if defines.is_empty() && env_defines.is_empty() && drop.is_empty() {
return None;
}
defines.sort_unstable();
env_defines.sort_unstable();
drop.sort_unstable();
drop.dedup();

let mut hasher = bun_wyhash::Wyhash::init(0);
let mut update = |bytes: &[u8]| {
hasher.update(&(bytes.len() as u64).to_le_bytes());
hasher.update(bytes);
};
// Separate sections: `init` lets a later env pair override a user pair.
for pairs in [defines, env_defines] {
update(&(pairs.len() as u64).to_le_bytes());
for (key, value) in pairs {
update(key);
update(value);
}
}
update(&(drop.len() as u64).to_le_bytes());
for item in drop {
update(item);
}
Some(hasher.final_())
}

pub(crate) fn for_identifier(&self, name: &[u8]) -> Option<&IdentifierDefine> {
if let Some(data) = self.identifiers.get(name) {
return Some(data);
Expand Down
1 change: 1 addition & 0 deletions src/js_parser/parse/parse_entry.rs
Original file line number Diff line number Diff line change
Expand Up @@ -212,6 +212,7 @@ impl<'a> Options<'a> {
runtime_transpiler_cache: None,
lower_using: f.lower_using,
bundler_feature_flags: None,
define_hash: f.define_hash,
repl_mode: f.repl_mode,
jsx_optimization_inline: f.jsx_optimization_inline,
},
Expand Down
10 changes: 10 additions & 0 deletions src/js_parser/parser.rs
Original file line number Diff line number Diff line change
Expand Up @@ -257,6 +257,9 @@ pub mod Runtime {
/// in watch/dev-server mode.
pub bundler_feature_flags: Option<Box<StringSet>>,

/// `Define::user_hash` of this parse's define table (runtime transpiler cache key).
pub define_hash: Option<u64>,

/// REPL mode: transforms code for interactive evaluation
/// - Wraps lone object literals `{...}` in parentheses
/// - Hoists variable declarations for REPL persistence
Expand Down Expand Up @@ -304,6 +307,7 @@ pub mod Runtime {
runtime_transpiler_cache: None,
lower_using: true,
bundler_feature_flags: None,
define_hash: None,
repl_mode: false,
jsx_optimization_inline: false,
}
Expand Down Expand Up @@ -391,6 +395,12 @@ pub mod Runtime {
hasher.update(b"\x00");
}
}

// Define pairs and `--drop` entries. `None` adds nothing, like an empty flag set.
if let Some(define_hash) = self.define_hash {
hasher.update(b"define");
hasher.update(&define_hash.to_le_bytes());
}
}

pub(crate) fn should_unwrap_require(&self, package_name: &[u8]) -> bool {
Expand Down
3 changes: 2 additions & 1 deletion src/jsc/RuntimeTranspilerCache.rs
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,8 @@ bun_core::declare_scope!(cache, visible);
/// offsets picked by a header byte) plus a body of tagged records with
/// u8/u16/u32 ids and implied slots dropped, instead of fixed u32 arrays.
/// Version 27: ModuleInfo string table holds Latin-1 / UTF-16 bodies, not WTF-8.
const EXPECTED_VERSION: u32 = 27;
/// Version 28: the define table and `--drop` entries participate in the features hash.
const EXPECTED_VERSION: u32 = 28;

/// Source files smaller than this are not written to / read from the on-disk
/// transpiler cache. Originally 50 KiB, which excluded almost every file in a
Expand Down
7 changes: 0 additions & 7 deletions src/runtime/cli/Arguments.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1569,13 +1569,6 @@ pub(crate) fn parse(cmd: CommandTag, ctx: Context<'_>) -> crate::Result<api::Tra
};
}
}

if let Some(define) = &opts.define {
if !define.keys.is_empty() {
bun_jsc::runtime_transpiler_cache::IS_DISABLED
.store(true, std::sync::atomic::Ordering::Relaxed);
}
}
}

if matches!(
Expand Down
107 changes: 107 additions & 0 deletions test/cli/run/transpiler-cache.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -320,6 +320,113 @@ describe("transpiler cache", () => {
expect(newCacheCount()).toBe(0); // cache hit, order doesn't matter
});

// A define replaces an identifier at parse time, so the define table is part
// of the cache key. The cache is shared by every project of the user, and
// keyed by source bytes, so two projects with the same file must not see
// each other's values.
describe("defines are part of the cache key", () => {
const code = `console.log(typeof BVAL === "undefined" ? "undefined" : BVAL);`;
const filler = Buffer.alloc((50 * 1024 * 1.5) | 0, "/").toString();

const run = (cwd: string, args: string[]) => {
const result = Bun.spawnSync({ cmd: [bunExe(), ...args], cwd, env });
if (!result.success) throw new Error(result.stderr.toString());
return result.stdout.toString().trim();
};

test("bunfig [define] invalidates cache", () => {
// `bun run <file>` loads bunfig.toml after the command line is parsed,
// so the define table only exists once the runtime is up.
const projectA = join(temp_dir, "a");
const projectB = join(temp_dir, "b");
for (const dir of [projectA, projectB]) {
mkdirSync(dir);
writeFileSync(join(dir, "a.js"), code + "\n//" + filler);
}
const setDefine = (dir: string, value: string | null) => {
if (value === null) rmSync(join(dir, "bunfig.toml"), { force: true });
else writeFileSync(join(dir, "bunfig.toml"), `[define]\nBVAL = '${JSON.stringify(value)}'\n`);
};

setDefine(projectA, "one");
expect(run(projectA, ["run", "./a.js"])).toBe("one");
expect(newCacheCount()).toBe(1);
expect(run(projectA, ["run", "./a.js"])).toBe("one");
expect(newCacheCount()).toBe(0);

// A new value: features_hash differs -> old entry deleted, new entry written
setDefine(projectA, "two");
expect(run(projectA, ["run", "./a.js"])).toBe("two");
expect(newCacheCount()).toBe(0);

setDefine(projectA, null);
expect(run(projectA, ["run", "./a.js"])).toBe("undefined");
expect(newCacheCount()).toBe(0);

// The same source bytes in another project, with its own define
setDefine(projectB, "mine");
expect(run(projectB, ["run", "./a.js"])).toBe("mine");
expect(newCacheCount()).toBe(0);
expect(run(projectB, ["./a.js"])).toBe("mine");
expect(newCacheCount()).toBe(0);
});

test("--define invalidates cache", () => {
writeFileSync(join(temp_dir, "a.js"), code + "\n//" + filler);

expect(run(temp_dir, ["--define", 'BVAL:"cli"', "a.js"])).toBe("cli");
expect(existsSync(cache_dir)).toBeTrue();
expect(newCacheCount()).toBe(1);
expect(run(temp_dir, ["--define", 'BVAL:"cli"', "a.js"])).toBe("cli");
expect(newCacheCount()).toBe(0);

expect(run(temp_dir, ["--define", 'BVAL:"other"', "a.js"])).toBe("other");
expect(newCacheCount()).toBe(0);

expect(run(temp_dir, ["a.js"])).toBe("undefined");
expect(newCacheCount()).toBe(0);

expect(run(temp_dir, ["run", "--define", 'BVAL:"cli"', "./a.js"])).toBe("cli");
expect(newCacheCount()).toBe(0);

// The key is built from the resolved map: flag order does not matter,
// and a key given twice keeps its last value, so `x` then `cli` is
// served the entry that `--define BVAL:"cli"` alone wrote above.
expect(run(temp_dir, ["--define", 'BVAL:"cli"', "--define", "OTHER:1", "a.js"])).toBe("cli");
expect(newCacheCount()).toBe(0);
const entry = join(cache_dir, readdirSync(cache_dir)[0]);
const written = readFileSync(entry);
expect(run(temp_dir, ["--define", "OTHER:1", "--define", 'BVAL:"cli"', "a.js"])).toBe("cli");
expect(readFileSync(entry).equals(written)).toBeTrue();

expect(run(temp_dir, ["--define", 'BVAL:"cli"', "a.js"])).toBe("cli");
const alone = readFileSync(entry);
expect(alone.equals(written)).toBeFalse();
expect(run(temp_dir, ["--define", 'BVAL:"x"', "--define", 'BVAL:"cli"', "a.js"])).toBe("cli");
expect(readFileSync(entry).equals(alone)).toBeTrue();
expect(newCacheCount()).toBe(0);
});

test("--drop invalidates cache", () => {
writeFileSync(
join(temp_dir, "a.js"),
`console.log("logged");\nprocess.stdout.write("written\\n");` + "\n//" + filler,
);

expect(run(temp_dir, ["a.js"])).toBe("logged\nwritten");
expect(newCacheCount()).toBe(1);

expect(run(temp_dir, ["--drop=console", "a.js"])).toBe("written");
expect(newCacheCount()).toBe(0);

expect(run(temp_dir, ["--drop=console", "a.js"])).toBe("written");
expect(newCacheCount()).toBe(0);

expect(run(temp_dir, ["a.js"])).toBe("logged\nwritten");
expect(newCacheCount()).toBe(0);
});
});

// Serving the entry point from the cache must not change how the modules it
// loads are resolved. Both of these are gated on the `has_loaded` flag, which
// used to be set only on the path that runs the printer.
Expand Down
Loading