Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
94 changes: 85 additions & 9 deletions src/jsc/bindings/NodeVM.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,10 @@
#include "JavaScriptCore/JSGlobalProxyInlines.h"
#include "GCDefferalContext.h"
#include "JSBuffer.h"
#include "xxhash3.h"

#include <wtf/MallocSpan.h>
#include <JavaScriptCore/JSCBytecodeCacheVersion.h>

#include <JavaScriptCore/DOMJITAbstractHeap.h>
#include <JavaScriptCore/DFGAbstractHeap.h>
Expand Down Expand Up @@ -127,7 +131,7 @@ bool extractCachedData(JSValue cachedDataValue, WTF::Vector<uint8_t>& outCachedD
return false;
}

Ref<JSC::CachedBytecode> createOwnedCachedBytecode(std::span<const uint8_t> bytes)
static Ref<JSC::CachedBytecode> createOwnedCachedBytecode(std::span<const uint8_t> bytes)
{
// UnlinkedFunctionExecutable's Decoder constructor (CachedTypes.cpp) keeps the Decoder and a
// payload offset, and decodes the body on the function's first call, borrowed payload or not.
Expand All @@ -136,6 +140,74 @@ Ref<JSC::CachedBytecode> createOwnedCachedBytecode(std::span<const uint8_t> byte
return JSC::CachedBytecode::create(WTF::move(payload), {});
}

// Integrity header verified by unwrapCachedData before decodeCodeBlock sees the payload; same role as V8's SerializedCodeData header.
struct CachedDataHeader {
uint32_t magic;
uint32_t payloadLength;
uint32_t sourceHash;
uint32_t jscVersion;
uint64_t payloadHash;
};
static_assert(sizeof(CachedDataHeader) == 24, "header layout is part of the cachedData format");

static constexpr uint32_t cachedDataMagic = 0x436E7542; // "BunC"

static uint64_t hashCachedDataPayload(std::span<const uint8_t> payload)
{
return highway_xxhash3_64(payload.data(), payload.size(), 0);
}

JSC::JSUint8Array* createCachedDataBuffer(JSGlobalObject* globalObject, const JSC::SourceCode& source, std::span<const uint8_t> bytecode)
{
JSC::JSUint8Array* buffer = WebCore::createUninitializedBuffer(globalObject, sizeof(CachedDataHeader) + bytecode.size());
if (!buffer) [[unlikely]] {
return nullptr;
}

const CachedDataHeader header {
.magic = cachedDataMagic,
.payloadLength = static_cast<uint32_t>(bytecode.size()),
.sourceHash = source.hash(),
.jscVersion = JSC::computeJSCBytecodeCacheVersion(),
.payloadHash = hashCachedDataPayload(bytecode),
};
uint8_t* data = buffer->typedVector();
memcpy(data, &header, sizeof(header));
if (!bytecode.empty()) {
memcpy(data + sizeof(header), bytecode.data(), bytecode.size());
}
return buffer;
}

RefPtr<JSC::CachedBytecode> unwrapCachedData(const JSC::SourceCode& source, std::span<const uint8_t> cachedData)
{
if (cachedData.size() < sizeof(CachedDataHeader)) {
return nullptr;
}

CachedDataHeader header;
memcpy(&header, cachedData.data(), sizeof(header));
if (header.magic != cachedDataMagic) {
return nullptr;
}

std::span<const uint8_t> payload = cachedData.subspan(sizeof(CachedDataHeader));
if (payload.size() != header.payloadLength) {
return nullptr;
}
if (header.sourceHash != source.hash()) {
return nullptr;
}
if (header.jscVersion != JSC::computeJSCBytecodeCacheVersion()) {
return nullptr;
}
if (header.payloadHash != hashCachedDataPayload(payload)) {
return nullptr;
Comment thread
coderabbitai[bot] marked this conversation as resolved.
}

return createOwnedCachedBytecode(payload);
}

JSC::JSFunction* constructAnonymousFunction(JSC::JSGlobalObject* globalObject, const ArgList& args, const SourceOrigin& sourceOrigin, CompileFunctionOptions&& options, JSC::SourceTaintedOrigin sourceTaintOrigin, JSC::JSScope* scope)
{
ASSERT(scope);
Expand Down Expand Up @@ -227,10 +299,13 @@ JSC::JSFunction* constructAnonymousFunction(JSC::JSGlobalObject* globalObject, c

TriState bytecodeAccepted = TriState::Indeterminate;

if (!options.cachedData.isEmpty()) {
cachedBytecode = createOwnedCachedBytecode(options.cachedData.span());
SourceCodeKey key(sourceCode, {}, JSC::SourceCodeType::ProgramType, lexicallyScopedFeatures, JSC::JSParserScriptMode::Classic, JSC::DerivedContextType::None, JSC::EvalContextType::None, false, {}, std::nullopt);
unlinkedProgramCodeBlock = JSC::decodeCodeBlock<UnlinkedProgramCodeBlock>(vm, key, *cachedBytecode);
// Node treats a provided-but-empty cachedData buffer as rejected, not absent.
if (options.cachedDataProvided) {
cachedBytecode = unwrapCachedData(sourceCode, options.cachedData.span());
if (cachedBytecode) {
SourceCodeKey key(sourceCode, {}, JSC::SourceCodeType::ProgramType, lexicallyScopedFeatures, JSC::JSParserScriptMode::Classic, JSC::DerivedContextType::None, JSC::EvalContextType::None, false, {}, std::nullopt);
unlinkedProgramCodeBlock = JSC::decodeCodeBlock<UnlinkedProgramCodeBlock>(vm, key, *cachedBytecode);
}
if (unlinkedProgramCodeBlock == nullptr) {
bytecodeAccepted = TriState::False;
} else {
Expand Down Expand Up @@ -271,7 +346,7 @@ JSC::JSFunction* constructAnonymousFunction(JSC::JSGlobalObject* globalObject, c
if (options.produceCachedData) {
RefPtr<JSC::CachedBytecode> producedBytecode = getBytecode(globalObject, JSC::SourceCodeType::ProgramType, sourceCode);
if (producedBytecode) {
JSC::JSUint8Array* buffer = WebCore::createBuffer(globalObject, producedBytecode->span());
JSC::JSUint8Array* buffer = createCachedDataBuffer(globalObject, sourceCode, producedBytecode->span());
RETURN_IF_EXCEPTION(throwScope, nullptr);
Bun::putDirectNamed(vm, function, "cachedData"_s, buffer);
Bun::putDirectNamed(vm, function, "cachedDataProduced"_s, jsBoolean(true));
Expand Down Expand Up @@ -491,8 +566,7 @@ JSC::EncodedJSValue createCachedData(JSGlobalObject* globalObject, const JSC::So
return throwVMError(globalObject, scope, "createCachedData failed"_s);
}

std::span<const uint8_t> bytes = bytecode->span();
JSC::JSUint8Array* buffer = WebCore::createBuffer(globalObject, bytes);
JSC::JSUint8Array* buffer = createCachedDataBuffer(globalObject, source, bytecode->span());
RETURN_IF_EXCEPTION(scope, {});

if (!buffer) {
Expand Down Expand Up @@ -1968,8 +2042,10 @@ bool CompileFunctionOptions::fromJS(JSC::JSGlobalObject* globalObject, JSC::VM&
// The validators return false both for "absent" and for "threw".
RETURN_IF_EXCEPTION(scope, false);

if (validateCachedData(globalObject, vm, scope, options, this->cachedData))
if (validateCachedData(globalObject, vm, scope, options, this->cachedData)) {
this->cachedDataProvided = true;
any = true;
}
RETURN_IF_EXCEPTION(scope, false);

JSValue parsingContextValue = options->getIfPropertyExists(globalObject, Identifier::fromString(vm, "parsingContext"_s));
Expand Down
5 changes: 3 additions & 2 deletions src/jsc/bindings/NodeVM.h
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,8 @@ namespace NodeVM {

RefPtr<JSC::CachedBytecode> getBytecode(JSGlobalObject* globalObject, JSC::SourceCodeType, const JSC::SourceCode& source);
bool extractCachedData(JSValue cachedDataValue, WTF::Vector<uint8_t>& outCachedData);
// Copies `bytes`: a decoded code block reads the payload long after decodeCodeBlock() returns.
Ref<JSC::CachedBytecode> createOwnedCachedBytecode(std::span<const uint8_t> bytes);
JSC::JSUint8Array* createCachedDataBuffer(JSGlobalObject* globalObject, const JSC::SourceCode& source, std::span<const uint8_t> bytecode);
RefPtr<JSC::CachedBytecode> unwrapCachedData(const JSC::SourceCode& source, std::span<const uint8_t> cachedData);
String stringifyAnonymousFunction(JSGlobalObject* globalObject, const ArgList& args, ThrowScope& scope, int* outOffset);
JSC::EncodedJSValue createCachedData(JSGlobalObject* globalObject, const JSC::SourceCode& source);
bool handleException(JSGlobalObject* globalObject, VM& vm, NakedPtr<JSC::Exception> exception, ThrowScope& throwScope);
Expand Down Expand Up @@ -71,6 +71,7 @@ class CompileFunctionOptions : public BaseVMOptions {
JSGlobalObject* parsingContext = nullptr;
JSValue contextExtensions {};
bool produceCachedData = false;
bool cachedDataProvided = false;

using BaseVMOptions::BaseVMOptions;

Expand Down
22 changes: 14 additions & 8 deletions src/jsc/bindings/NodeVMScript.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -66,8 +66,10 @@ bool ScriptOptions::fromJS(JSC::JSGlobalObject* globalObject, JSC::VM& vm, JSC::
any = true;
RETURN_IF_EXCEPTION(scope, false);

if (validateCachedData(globalObject, vm, scope, options, this->cachedData))
if (validateCachedData(globalObject, vm, scope, options, this->cachedData)) {
this->cachedDataProvided = true;
any = true;
}
RETURN_IF_EXCEPTION(scope, false);

// Handle importModuleDynamically option
Expand Down Expand Up @@ -169,17 +171,20 @@ constructScript(JSGlobalObject* globalObject, CallFrame* callFrame, JSValue newT

WTF::Vector<uint8_t>& cachedData = script->cachedData();

if (!cachedData.isEmpty()) {
// Node treats a provided-but-empty cachedData buffer as rejected, not absent.
if (script->options().cachedDataProvided) {
JSC::ProgramExecutable* executable = script->cachedExecutable();
if (!executable) {
executable = script->createExecutable();
}
ASSERT(executable);

JSC::LexicallyScopedFeatures lexicallyScopedFeatures = globalObject->globalScopeExtension() ? JSC::TaintedByWithScopeLexicallyScopedFeature : JSC::NoLexicallyScopedFeatures;
JSC::SourceCodeKey key(script->source(), {}, JSC::SourceCodeType::ProgramType, lexicallyScopedFeatures, JSC::JSParserScriptMode::Classic, JSC::DerivedContextType::None, JSC::EvalContextType::None, false, {}, std::nullopt);
Ref<JSC::CachedBytecode> cachedBytecode = NodeVM::createOwnedCachedBytecode(cachedData.span());
JSC::UnlinkedProgramCodeBlock* unlinkedBlock = JSC::decodeCodeBlock<UnlinkedProgramCodeBlock>(vm, key, WTF::move(cachedBytecode));
JSC::UnlinkedProgramCodeBlock* unlinkedBlock = nullptr;
if (RefPtr<JSC::CachedBytecode> cachedBytecode = unwrapCachedData(script->source(), cachedData.span())) {
JSC::LexicallyScopedFeatures lexicallyScopedFeatures = globalObject->globalScopeExtension() ? JSC::TaintedByWithScopeLexicallyScopedFeature : JSC::NoLexicallyScopedFeatures;
JSC::SourceCodeKey key(script->source(), {}, JSC::SourceCodeType::ProgramType, lexicallyScopedFeatures, JSC::JSParserScriptMode::Classic, JSC::DerivedContextType::None, JSC::EvalContextType::None, false, {}, std::nullopt);
unlinkedBlock = JSC::decodeCodeBlock<UnlinkedProgramCodeBlock>(vm, key, cachedBytecode.releaseNonNull());
}
Comment thread
claude[bot] marked this conversation as resolved.
Comment thread
robobun marked this conversation as resolved.

if (!unlinkedBlock) {
script->cachedDataRejected(TriState::True);
Expand All @@ -200,6 +205,8 @@ constructScript(JSGlobalObject* globalObject, CallFrame* callFrame, JSValue newT
script->cachedDataRejected(TriState::True);
}
}
// unwrapCachedData owns its copy; nothing reads m_options.cachedData after this.
cachedData = {};
} else if (script->options().produceCachedData)
script->cacheBytecode();

Expand Down Expand Up @@ -266,8 +273,7 @@ JSC::JSUint8Array* NodeVMScript::getBytecodeBuffer()
if (!m_cachedBytecode)
return nullptr;

std::span<const uint8_t> bytes = m_cachedBytecode->span();
m_cachedBytecodeBuffer.set(vm(), this, WebCore::createBuffer(globalObject(), bytes));
m_cachedBytecodeBuffer.set(vm(), this, createCachedDataBuffer(globalObject(), m_source, m_cachedBytecode->span()));
RETURN_IF_EXCEPTION(scope, nullptr);
if (!m_cachedBytecodeBuffer) {
return nullptr;
Expand Down
2 changes: 1 addition & 1 deletion src/jsc/bindings/NodeVMScript.h
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ class ScriptOptions : public BaseVMOptions {
WTF::Vector<uint8_t> cachedData;
std::optional<int64_t> timeout = std::nullopt;
bool produceCachedData = false;
bool cachedDataProvided = false;

using BaseVMOptions::BaseVMOptions;

Expand Down Expand Up @@ -77,7 +78,6 @@ class NodeVMScript final : public JSC::JSDestructibleObject {
WTF::Vector<uint8_t>& cachedData() { return m_options.cachedData; }
JSC::ProgramExecutable* cachedExecutable() const { return m_cachedExecutable.get(); }
bool cachedDataProduced() const { return m_cachedDataProduced; }
void cachedDataProduced(bool value) { m_cachedDataProduced = value; }
TriState cachedDataRejected() const { return m_cachedDataRejected; }
void cachedDataRejected(TriState value) { m_cachedDataRejected = value; }

Expand Down
22 changes: 14 additions & 8 deletions src/jsc/bindings/NodeVMSourceTextModule.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,8 @@ NodeVMSourceTextModule* NodeVMSourceTextModule::create(VM& vm, JSGlobalObject* g
WTF::move(sourceCode), moduleWrapper, initializeImportMeta);
ptr->finishCreation(vm);

if (cachedData.isEmpty()) {
// Node treats a provided-but-empty cachedData buffer as rejected, not absent.
if (cachedDataValue.isUndefined()) {
return ptr;
}

Expand All @@ -126,11 +127,12 @@ NodeVMSourceTextModule* NodeVMSourceTextModule::create(VM& vm, JSGlobalObject* g

// Decoding checks the format and the source key. Linking would need
// the module's JSModuleEnvironment, which does not exist yet.
LexicallyScopedFeatures lexicallyScopedFeatures = StrictModeLexicallyScopedFeature;
SourceCodeKey key(ptr->sourceCode(), {}, SourceCodeType::ModuleType, lexicallyScopedFeatures, JSParserScriptMode::Module, DerivedContextType::None, EvalContextType::None, false, {}, std::nullopt);
Ref<CachedBytecode> cachedBytecode = NodeVM::createOwnedCachedBytecode(cachedData.span());
if (decodeCodeBlock<UnlinkedModuleProgramCodeBlock>(vm, key, WTF::move(cachedBytecode)))
return ptr;
if (RefPtr<CachedBytecode> cachedBytecode = unwrapCachedData(ptr->sourceCode(), cachedData.span())) {
LexicallyScopedFeatures lexicallyScopedFeatures = StrictModeLexicallyScopedFeature;
SourceCodeKey key(ptr->sourceCode(), {}, SourceCodeType::ModuleType, lexicallyScopedFeatures, JSParserScriptMode::Module, DerivedContextType::None, EvalContextType::None, false, {}, std::nullopt);
if (decodeCodeBlock<UnlinkedModuleProgramCodeBlock>(vm, key, cachedBytecode.releaseNonNull()))
return ptr;
}

throwError(globalObject, scope, ErrorCode::ERR_VM_MODULE_CACHED_DATA_REJECTED, "cachedData buffer was rejected"_s);
return nullptr;
Expand Down Expand Up @@ -502,8 +504,12 @@ JSUint8Array* NodeVMSourceTextModule::cachedData(JSGlobalObject* globalObject)
if (!m_cachedBytecodeBuffer) {
RefPtr<CachedBytecode> cachedBytecode = bytecode(globalObject);
RETURN_IF_EXCEPTION(scope, nullptr);
std::span<const uint8_t> bytes = cachedBytecode->span();
JSUint8Array* buffer = WebCore::createBuffer(globalObject, bytes);
// getBytecode can return null without throwing (serialization failure).
if (!cachedBytecode) [[unlikely]] {
throwVMError(globalObject, scope, "createCachedData failed"_s);
return nullptr;
}
JSUint8Array* buffer = createCachedDataBuffer(globalObject, m_sourceCode, cachedBytecode->span());
RETURN_IF_EXCEPTION(scope, nullptr);
m_cachedBytecodeBuffer.set(vm, this, buffer);
Comment thread
coderabbitai[bot] marked this conversation as resolved.
}
Expand Down
46 changes: 34 additions & 12 deletions test/bundler/bundler_bytecode_portable.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -382,20 +382,33 @@ function fingerprint(bytecode: Uint8Array, isPayload = true) {
return { sha256: Bun.CryptoHasher.hash("sha256", copy, "hex"), bytes: copy.byteLength };
}

// node:vm cachedData wraps the JSC payload in a 24-byte integrity header (CachedDataHeader in
// src/jsc/bindings/NodeVM.cpp) whose version and checksum fields also move on every WebKit upgrade. Strip it so the
// vm entries pin the serialized bytecode alone, like the bundler outputs.
function vmPayload(cachedData: Uint8Array) {
const headerSize = 24;
const payloadLength = new DataView(cachedData.buffer, cachedData.byteOffset, cachedData.byteLength).getUint32(
4,
true,
);
expect(cachedData.byteLength).toBe(headerSize + payloadLength);
return cachedData.subarray(headerSize);
}

describe("bytecode cache portability", () => {
test("encoder output is identical on every platform", async () => {
// The bundler builds are separate processes: start them all, then encode the in-process cases while they run.
const bundled = Promise.all(bundlerBuilds.map(build));
const outputs: Record<string, unknown> = {};
// Program and module code blocks straight from the encoder, without the bundler in between.
outputs["vm.Script features.js"] = fingerprint(
new vm.Script(featuresSource, { filename: "features.js", produceCachedData: true }).cachedData!,
vmPayload(new vm.Script(featuresSource, { filename: "features.js", produceCachedData: true }).cachedData!),
);
outputs["vm.Script shapes.js"] = fingerprint(
new vm.Script(shapesSource(), { filename: "shapes.js", produceCachedData: true }).cachedData!,
vmPayload(new vm.Script(shapesSource(), { filename: "shapes.js", produceCachedData: true }).cachedData!),
);
outputs["vm.Script records.js"] = fingerprint(
new vm.Script(recordsSource, { filename: "records.js", produceCachedData: true }).cachedData!,
vmPayload(new vm.Script(recordsSource, { filename: "records.js", produceCachedData: true }).cachedData!),
);
// A builtin (what `bun build --compile --bytecode` embeds for node:* / bun:* modules): @-intrinsics and the
// builtin-executable entry, which user source never produces. Bun's own internal modules are not hashed here because
Expand All @@ -404,26 +417,35 @@ describe("bytecode cache portability", () => {
outputs["builtin corpus"] = fingerprint(builtin.bytecode);
outputs["builtin corpus strings"] = fingerprint(builtin.strings, false); // the external string table --compile embeds beside it
outputs["vm.SourceTextModule module.js"] = fingerprint(
new vm.SourceTextModule(moduleSource, { identifier: "module.js" }).createCachedData(),
vmPayload(new vm.SourceTextModule(moduleSource, { identifier: "module.js" }).createCachedData()),
);
outputs["vm.Script big.js"] = fingerprint(
new vm.Script(bigSource(), { filename: "big.js", produceCachedData: true }).cachedData!,
vmPayload(new vm.Script(bigSource(), { filename: "big.js", produceCachedData: true }).cachedData!),
);
outputs["vm.Script source-forms.js"] = fingerprint(
new vm.Script(sourceFormsSource(), { filename: "source-forms.js", produceCachedData: true }).cachedData!,
vmPayload(
new vm.Script(sourceFormsSource(), { filename: "source-forms.js", produceCachedData: true }).cachedData!,
),
);
const librarySource = (lib: string) => readFileSync(join(corpusDir, "../../node_modules", lib), "utf8");
outputs["vm.Script lodash.js"] = fingerprint(
new vm.Script(librarySource("lodash/lodash.js"), { filename: "lodash.js", produceCachedData: true }).cachedData!,
vmPayload(
new vm.Script(librarySource("lodash/lodash.js"), { filename: "lodash.js", produceCachedData: true })
.cachedData!,
),
);
outputs["vm.Script typescript.js"] = fingerprint(
new vm.Script(librarySource("typescript/lib/typescript.js"), {
filename: "typescript.js",
produceCachedData: true,
}).cachedData!,
vmPayload(
new vm.Script(librarySource("typescript/lib/typescript.js"), {
filename: "typescript.js",
produceCachedData: true,
}).cachedData!,
),
);
outputs["vm.SourceTextModule acorn.mjs"] = fingerprint(
new vm.SourceTextModule(librarySource("acorn/dist/acorn.mjs"), { identifier: "acorn.mjs" }).createCachedData(),
vmPayload(
new vm.SourceTextModule(librarySource("acorn/dist/acorn.mjs"), { identifier: "acorn.mjs" }).createCachedData(),
),
);
for (const [i, { name }] of bundlerBuilds.entries()) {
const { js, jsc } = (await bundled)[i];
Expand Down
Loading
Loading