diff --git a/Source/JavaScriptCore/bytecode/UnlinkedFunctionExecutable.cpp b/Source/JavaScriptCore/bytecode/UnlinkedFunctionExecutable.cpp index d77230c3e4f68..9fb0d95632af3 100644 --- a/Source/JavaScriptCore/bytecode/UnlinkedFunctionExecutable.cpp +++ b/Source/JavaScriptCore/bytecode/UnlinkedFunctionExecutable.cpp @@ -261,8 +261,24 @@ UnlinkedFunctionCodeBlock* UnlinkedFunctionExecutable::unlinkedCodeBlockFor( VM& vm, const SourceCode& source, CodeSpecializationKind specializationKind, OptionSet codeGenerationMode, ParserError& error, SourceParseMode parseMode, OptimizeBytecode optimize) { - if (m_isCached) + if (m_isCached) { +#if USE(BUN_JSC_ADDITIONS) + // Code of a payload that outlives the program, about to be run: what a payload order file is about. (Not what + // codeBlocksDecodingCached decodes, for a link; the callers that generate code without running it, CodeCache's and + // the shell's, have executables that were just parsed.) + auto* recorder = BytecodeOrderRecorder::ofVM(vm); + std::optional recordedSource; + if (recorder && m_decoder->canBorrowPayload()) [[unlikely]] + recordedSource = m_decoder->orderSource(); +#endif decodeCachedCodeBlocks(vm); +#if USE(BUN_JSC_ADDITIONS) + if (recordedSource) [[unlikely]] { + if (auto key = orderFunctionKey(*this, source)) + recorder->didDecodeFunction(*recordedSource, *key); + } +#endif + } switch (specializationKind) { case CodeSpecializationKind::CodeForCall: if (UnlinkedFunctionCodeBlock* codeBlock = m_unlinkedCodeBlockForCall.get()) @@ -295,6 +311,15 @@ UnlinkedFunctionCodeBlock* UnlinkedFunctionExecutable::unlinkedCodeBlockFor( return result; } +#if USE(BUN_JSC_ADDITIONS) +std::pair UnlinkedFunctionExecutable::codeBlocksDecodingCached(VM& vm) +{ + if (m_isCached) + decodeCachedCodeBlocks(vm); + return { m_unlinkedCodeBlockForCall.get(), m_unlinkedCodeBlockForConstruct.get() }; +} +#endif + void UnlinkedFunctionExecutable::decodeCachedCodeBlocks(VM& vm) { ASSERT(m_isCached); diff --git a/Source/JavaScriptCore/bytecode/UnlinkedFunctionExecutable.h b/Source/JavaScriptCore/bytecode/UnlinkedFunctionExecutable.h index f5e834c291e8d..056ea66b2357c 100644 --- a/Source/JavaScriptCore/bytecode/UnlinkedFunctionExecutable.h +++ b/Source/JavaScriptCore/bytecode/UnlinkedFunctionExecutable.h @@ -213,6 +213,11 @@ class UnlinkedFunctionExecutable final : public JSCell { bool isBuiltinFunction() const { return m_isBuiltinFunction; } // The code blocks are still (or again) in the bytecode cache this was decoded from. bool isCached() const { return m_isCached; } +#if USE(BUN_JSC_ADDITIONS) + // The code blocks for call and construct as they are once whatever the bytecode cache holds for this function is + // decoded; never generates one (unlinkedCodeBlockFor does). + JS_EXPORT_PRIVATE std::pair codeBlocksDecodingCached(VM&); +#endif ConstructAbility constructAbility() const { return static_cast(m_constructAbility); } JSParserScriptMode scriptMode() const { return static_cast(m_scriptMode); } bool isClassConstructorFunction() const @@ -229,6 +234,12 @@ class UnlinkedFunctionExecutable final : public JSCell { } bool isClass() const { return m_isClass; } bool isBuiltinDefaultClassConstructor() const { return m_isBuiltinDefaultClassConstructor; } +#if USE(BUN_JSC_ADDITIONS) + // Where classSource() starts and where the first of classElementDefinitions() is, for orderFunctionKey: read out + // of the cached record as they lie, so that nothing of a function that was decoded from a cache is materialized. + std::optional classSourceStartWithoutMaterializing() const; + std::optional firstClassElementOffsetWithoutMaterializing() const; +#endif RefPtr parentScopeTDZVariables() const { diff --git a/Source/JavaScriptCore/debugger/Debugger.cpp b/Source/JavaScriptCore/debugger/Debugger.cpp index 22eeb4778308f..609a20241275b 100644 --- a/Source/JavaScriptCore/debugger/Debugger.cpp +++ b/Source/JavaScriptCore/debugger/Debugger.cpp @@ -562,7 +562,7 @@ void Debugger::toggleBreakpoint(Breakpoint& breakpoint, Debugger::BreakpointStat void Debugger::recompileAllJSFunctions() { - m_vm.deleteAllCode(PreventCollectionAndDeleteAllCode); + m_vm.deleteAllCodeToGenerateItAgain(PreventCollectionAndDeleteAllCode); } DebuggerParseData& Debugger::debuggerParseData(SourceID sourceID, SourceProvider* provider) diff --git a/Source/JavaScriptCore/heap/Heap.cpp b/Source/JavaScriptCore/heap/Heap.cpp index a2fdccd539328..a3d1154a7f6e4 100644 --- a/Source/JavaScriptCore/heap/Heap.cpp +++ b/Source/JavaScriptCore/heap/Heap.cpp @@ -1205,6 +1205,9 @@ void Heap::deleteAllUnlinkedCodeBlocks(DeleteAllCodeEffort effort, OptionSetvm(); + // Shared Baseline code, below, still goes. + if (vm.keepsUnlinkedCode()) [[unlikely]] + which.remove({ UnlinkedCodeToDelete::Generated, UnlinkedCodeToDelete::RecoverableFromCache }); PreventCollectionScope preventCollectionScope(*this); RELEASE_ASSERT(!m_collectionScope); diff --git a/Source/JavaScriptCore/inspector/agents/InspectorRuntimeAgent.cpp b/Source/JavaScriptCore/inspector/agents/InspectorRuntimeAgent.cpp index bd5258cf60b70..4717649a176ad 100644 --- a/Source/JavaScriptCore/inspector/agents/InspectorRuntimeAgent.cpp +++ b/Source/JavaScriptCore/inspector/agents/InspectorRuntimeAgent.cpp @@ -508,7 +508,7 @@ void InspectorRuntimeAgent::setTypeProfilerEnabledState(bool isTypeProfilingEnab vm.whenIdle([&vm, isTypeProfilingEnabled] () { bool shouldRecompileFromTypeProfiler = (isTypeProfilingEnabled ? vm.enableTypeProfiler() : vm.disableTypeProfiler()); if (shouldRecompileFromTypeProfiler) - vm.deleteAllCode(PreventCollectionAndDeleteAllCode); + vm.deleteAllCodeToGenerateItAgain(PreventCollectionAndDeleteAllCode); }); } @@ -523,7 +523,7 @@ void InspectorRuntimeAgent::setControlFlowProfilerEnabledState(bool isControlFlo bool shouldRecompileFromControlFlowProfiler = (isControlFlowProfilingEnabled ? vm.enableControlFlowProfiler() : vm.disableControlFlowProfiler()); if (shouldRecompileFromControlFlowProfiler) - vm.deleteAllCode(PreventCollectionAndDeleteAllCode); + vm.deleteAllCodeToGenerateItAgain(PreventCollectionAndDeleteAllCode); }); } diff --git a/Source/JavaScriptCore/runtime/CachedBytecode.cpp b/Source/JavaScriptCore/runtime/CachedBytecode.cpp index d2272772dbc98..d8bcae2064b5d 100644 --- a/Source/JavaScriptCore/runtime/CachedBytecode.cpp +++ b/Source/JavaScriptCore/runtime/CachedBytecode.cpp @@ -32,6 +32,7 @@ #include "UnlinkedCodeBlock.h" #include "UnlinkedFunctionExecutable.h" #include "WeakInlines.h" +#include #include WTF_ALLOW_UNSAFE_BUFFER_USAGE_BEGIN @@ -40,6 +41,142 @@ namespace JSC { WTF_MAKE_TZONE_ALLOCATED_IMPL(PersistentBytecodePayloads); +#if USE(BUN_JSC_ADDITIONS) +WTF_MAKE_TZONE_ALLOCATED_IMPL(BytecodeOrderRecorder); + +BytecodeOrderRecorder::BytecodeOrderRecorder(bool isOver) + : m_isOver(isOver) +{ +} +BytecodeOrderRecorder::~BytecodeOrderRecorder() = default; + +static Lock s_orderRecordersLock; +static bool s_orderRecordingIsOver WTF_GUARDED_BY_LOCK(s_orderRecordersLock) { false }; +static Vector>& orderRecorders() WTF_REQUIRES_LOCK(s_orderRecordersLock) +{ + static NeverDestroyed>> recorders; + return recorders; +} + +Ref BytecodeOrderRecorder::create() +{ + Locker locker { s_orderRecordersLock }; + Ref recorder = adoptRef(*new BytecodeOrderRecorder(s_orderRecordingIsOver)); + orderRecorders().append(recorder); + return recorder; +} + +Vector> BytecodeOrderRecorder::endRecordingInProcess() +{ + Locker locker { s_orderRecordersLock }; + s_orderRecordingIsOver = true; + return orderRecorders(); +} + +BytecodeOrderRecorder* BytecodeOrderRecorder::ofVM(VM& vm) +{ + auto* payloads = vm.persistentBytecodePayloadsIfExists(); + return payloads ? payloads->orderRecorderIfRecording() : nullptr; +} + +unsigned BytecodeOrderRecorder::indexOf(RecordedOrderSource source) +{ + // Most of what is decoded comes out of the source the last thing came out of. + if (m_lastSource < m_recorded.sources.size() && m_recorded.sources[m_lastSource] == source) + return m_lastSource; + auto result = m_sources.add({ source.payload, source.entryOffset }, m_recorded.sources.size()); + if (result.isNewEntry) + m_recorded.sources.append(source); + m_lastSource = result.iterator->value; + return m_lastSource; +} + +void BytecodeOrderRecorder::didDecodeFunction(RecordedOrderSource source, OrderFunctionKey key) +{ + if (isOver()) + return; + Locker locker { m_lock }; + if (m_isOver) + return; + unsigned index = indexOf(source); + // Two bits of kind, 32 of start, 30 of source; the top two values are the table's own. + RELEASE_ASSERT(index < (1u << 30) - 1); + if (m_seenFunctions.add(static_cast(index) << 34 | static_cast(key.start) << 2 | static_cast(key.kind)).isNewEntry) + m_recorded.functions.append({ index, key }); +} + +void BytecodeOrderRecorder::didDecodeModule(RecordedOrderSource source) +{ + if (isOver()) + return; + Locker locker { m_lock }; + if (m_isOver) + return; + unsigned index = indexOf(source); + if (!m_seenModules.set(index)) + m_recorded.modules.append(index); +} + +void BytecodeOrderRecorder::didRejectModule(RecordedOrderSource source) +{ + if (isOver()) + return; + Locker locker { m_lock }; + if (m_isOver) + return; + unsigned index = indexOf(source); + if (!m_seenRejectedModules.set(index)) + m_recorded.rejectedModules.append(index); +} + +void BytecodeOrderRecorder::didReadString(std::span stringTable, uint32_t ordinal) +{ + if (isOver()) + return; + Locker locker { m_lock }; + if (m_isOver || m_seenStrings.set(ordinal)) + return; + // Ordinals mean something in one table only: a VM has one for as long as it records. + RELEASE_ASSERT(m_recorded.stringOrdinals.isEmpty() || m_recorded.stringTable.data() == stringTable.data()); + m_recorded.stringTable = stringTable; + m_recorded.stringOrdinals.append(ordinal); +} + +auto BytecodeOrderRecorder::take() -> Snapshot +{ + Locker locker { m_lock }; + m_isOver.store(true); + m_sources.clear(); + m_seenFunctions.clear(); + return std::exchange(m_recorded, { }); +} + +void PersistentBytecodePayloads::setLinkedPayload(std::span payload, const std::array& regionEnds) +{ + m_linkedPayloadBase = payload.data(); + m_linkedPayloadStatistics = { }; + m_linkedPayloadStatistics.regionEnds = regionEnds; +} + +BytecodeOrderRecorder& PersistentBytecodePayloads::enableOrderRecording() +{ + if (!m_orderRecorder) { + m_orderRecorder = BytecodeOrderRecorder::create(); + // Each function's code is decoded once: a recording is the same whenever the collector runs. + m_vm.keepUnlinkedCodeUntil(m_orderRecorder->isOverFlag()); + } + return *m_orderRecorder; +} + +BytecodeOrderRecorder* PersistentBytecodePayloads::orderRecorderIfRecording() +{ + // (The process keeps every recorder for good: VM::keepUnlinkedCodeUntil.) + if (m_orderRecorder && m_orderRecorder->isOver()) [[unlikely]] + m_orderRecorder = nullptr; + return m_orderRecorder.get(); +} +#endif + PersistentBytecodePayloads::PersistentBytecodePayloads(VM& vm) : m_vm(vm) { diff --git a/Source/JavaScriptCore/runtime/CachedBytecode.h b/Source/JavaScriptCore/runtime/CachedBytecode.h index 8e107f0984880..cece177be219b 100644 --- a/Source/JavaScriptCore/runtime/CachedBytecode.h +++ b/Source/JavaScriptCore/runtime/CachedBytecode.h @@ -30,12 +30,17 @@ #include "ParserModes.h" #include "Weak.h" #include "WeakGCHashTable.h" -#include +#include +#include #include +#include +#include +#include #include #include #include #include +#include #include namespace JSC { @@ -43,6 +48,7 @@ namespace JSC { class Decoder; class SourceProvider; class UnlinkedCodeBlock; +class UnlinkedFunctionCodeBlock; class UnlinkedFunctionExecutable; class VM; @@ -84,6 +90,12 @@ class CachedBytecode : public RefCounted { // Where the root record starts within the payload (a function code block is written after its own arrays). size_t rootOffset() const { return m_rootOffset; } void setRootOffset(size_t offset) { m_rootOffset = offset; } +#if USE(BUN_JSC_ADDITIONS) + // Where this code's cache entry starts within the payload. Non-zero when several modules were encoded into one + // payload (BytecodeLinkEncoder): every offset a Decoder keeps stays relative to the start of the shared payload. + size_t entryOffset() const { return m_entryOffset; } + void setEntryOffset(size_t offset) { m_entryOffset = offset; } +#endif void setPayloadIsPersistent() { m_payload.setIsPersistent(); } bool payloadIsOwnedOrPersistent() const { return m_payload.isOwnedOrPersistent(); } bool hasUpdates() const { return !m_updates.isEmpty(); } @@ -102,10 +114,73 @@ class CachedBytecode : public RefCounted { size_t m_size { 0 }; CachePayload m_payload; size_t m_rootOffset { 0 }; +#if USE(BUN_JSC_ADDITIONS) + size_t m_entryOffset { 0 }; +#endif LeafExecutableMap m_leafExecutables; Vector m_updates; }; +#if USE(BUN_JSC_ADDITIONS) +// The regions of a payload written by BytecodeLinkEncoder, in the order they lie in it. +namespace BytecodeLinkRegions { +enum : unsigned { EarlyHeads, Hot, Unknown, LateHeads, Cold, ExpressionInfo, Count }; +} + +// What one VM read out of its persistent payloads, in first-use order: the input of a payload order file. Exists only +// while recording (PersistentBytecodePayloads::enableOrderRecording). Every recorder of the process stays registered, +// whether or not its VM is still alive, and any thread may take a snapshot of it (the thread that writes the order file +// when the process exits is not the thread of a Worker's VM), so what it remembers must not die with the VM: a source is +// where its payload is, and strings are ordinals into a table, both of which the embedder keeps for good. +class BytecodeOrderRecorder final : public ThreadSafeRefCounted { + WTF_MAKE_NONCOPYABLE(BytecodeOrderRecorder); + WTF_MAKE_TZONE_ALLOCATED(BytecodeOrderRecorder); +public: + static Ref create(); + ~BytecodeOrderRecorder(); + // Every recorder of the process, in creation order; one made from here on records nothing. + static Vector> endRecordingInProcess(); + // Null unless the VM records. + static BytecodeOrderRecorder* ofVM(VM&); + + // Only code of payloads that outlive the program (CachedBytecode::payloadIsPersistent) is what a payload order file + // lays out, and can be known by where it is; the callers see to that. + void didDecodeFunction(RecordedOrderSource, OrderFunctionKey); // decoded in order to be run, whether or not it then runs + void didDecodeModule(RecordedOrderSource); + void didRejectModule(RecordedOrderSource); // it has bytecode, which is not for the source it is run from + void didReadString(std::span stringTable, uint32_t ordinal); + + struct Snapshot { + Vector sources; + Vector functions; + Vector modules; // indices into `sources` + Vector rejectedModules; + std::span stringTable; // DecoderStringTable's bytes + Vector stringOrdinals; + }; + // What is decoded from here on is not recorded. + Snapshot take(); + bool isOver() const { return m_isOver.load(); } + const std::atomic& isOverFlag() const { return m_isOver; } + +private: + explicit BytecodeOrderRecorder(bool isOver); + unsigned indexOf(RecordedOrderSource) WTF_REQUIRES_LOCK(m_lock); + + Lock m_lock; + std::atomic m_isOver; // set with m_lock held + + Snapshot m_recorded WTF_GUARDED_BY_LOCK(m_lock); + UncheckedKeyHashMap, unsigned> m_sources WTF_GUARDED_BY_LOCK(m_lock); + unsigned m_lastSource WTF_GUARDED_BY_LOCK(m_lock) { 0 }; + // A function is decoded again after its code was returned to the cache. + OrderHashSet m_seenFunctions WTF_GUARDED_BY_LOCK(m_lock); + BitVector m_seenModules WTF_GUARDED_BY_LOCK(m_lock); + BitVector m_seenRejectedModules WTF_GUARDED_BY_LOCK(m_lock); + BitVector m_seenStrings WTF_GUARDED_BY_LOCK(m_lock); +}; +#endif + // The persistent payloads (CachePayload::isPersistent) a VM has code from. An UnlinkedCodeBlock decoded from one // remembers its slot here and its record's offset, which is enough to decode it again after it was dropped // (UnlinkedFunctionExecutable::returnCodeToCache) without every block holding on to a Decoder. @@ -148,6 +223,40 @@ class PersistentBytecodePayloads final : public WeakGCHashTable { // Before the heap's last finalization takes the weak references' storage. void clearChildExecutables() { m_childExecutables.clear(); } +#if USE(BUN_JSC_ADDITIONS) + JS_EXPORT_PRIVATE BytecodeOrderRecorder& enableOrderRecording(); + // Null once the recording was taken (BytecodeOrderRecorder::take). + BytecodeOrderRecorder* orderRecorderIfRecording(); + + // How well the order file a payload was laid out by (BytecodeLinkEncoder) matches what this VM runs: the function + // bodies decoded out of the payload, by the region they lie in. A body decoded again after its code was returned to + // the cache counts again. `bytes`: a body's own arrays and record, not what it shares with an identical earlier one. + struct LinkedPayloadStatistics { + struct Bodies { + uint64_t count { 0 }; + uint64_t bytes { 0 }; + }; + std::array regionEnds { }; + Bodies hot; + Bodies unknown; + Bodies cold; + }; + // The embedder says which of its payloads is a linked one. Nothing is counted for any other payload. + JS_EXPORT_PRIVATE void setLinkedPayload(std::span, const std::array& regionEnds); + const LinkedPayloadStatistics* linkedPayloadStatistics() const { return m_linkedPayloadBase ? &m_linkedPayloadStatistics : nullptr; } + void didDecodeFunctionBody(const void* payloadBase, uint32_t recordOffset, uint32_t bytes) + { + if (payloadBase != m_linkedPayloadBase) + return; + auto& regionEnds = m_linkedPayloadStatistics.regionEnds; + auto& bodies = recordOffset < regionEnds[BytecodeLinkRegions::Hot] ? m_linkedPayloadStatistics.hot + : recordOffset < regionEnds[BytecodeLinkRegions::Unknown] ? m_linkedPayloadStatistics.unknown + : m_linkedPayloadStatistics.cold; + bodies.count++; + bodies.bytes += bytes; + } +#endif + // For diagnostics: what this and the Decoders it can reach hold on to. struct Statistics { size_t payloads { 0 }; @@ -176,6 +285,11 @@ class PersistentBytecodePayloads final : public WeakGCHashTable { UncheckedKeyHashMap m_indices; VM& m_vm; UncheckedKeyHashMap>> m_childExecutables; +#if USE(BUN_JSC_ADDITIONS) + RefPtr m_orderRecorder; + const void* m_linkedPayloadBase { nullptr }; // never a Decoder's base while null + LinkedPayloadStatistics m_linkedPayloadStatistics; +#endif }; } // namespace JSC diff --git a/Source/JavaScriptCore/runtime/CachedTypes.cpp b/Source/JavaScriptCore/runtime/CachedTypes.cpp index 5908ceab10af7..43f7878a00893 100644 --- a/Source/JavaScriptCore/runtime/CachedTypes.cpp +++ b/Source/JavaScriptCore/runtime/CachedTypes.cpp @@ -32,12 +32,14 @@ #include "BuiltinNames.h" #include "BytecodeCacheError.h" #include "BytecodeLivenessAnalysis.h" +#include "CodeCache.h" #include "JSCBytecodeCacheVersion.h" #include "JSCInlines.h" #include "JSCellButterfly.h" #include "JSTemplateObjectDescriptor.h" #include "ScopedArgumentsTable.h" #include "SourceCodeKey.h" +#include "StrongInlines.h" #include "SourceProvider.h" #include "SymbolTableInlines.h" #include "UnlinkedEvalCodeBlock.h" @@ -152,6 +154,95 @@ void Decoder::setAtomForOrdinal(uint32_t ordinal, AtomStringImpl& atom) } // 1- and 2-character inline strings are the bulk of minified identifiers: length 1 is SmallStrings' single-character reps; length 2 hits one lazy 64x64 table on the VM, indexed by identifier character class (shared by every Decoder, not one per retained Decoder); length 3 (module_info's minified import/export names, once past two characters), and length 2 with a character outside those classes, a small direct-mapped cache in front of the atom table. +#if USE(BUN_JSC_ADDITIONS) +namespace { + +// FNV-1a, finished with a 64-bit mix. The top two values are reserved for hash-table sentinels. +class OrderHasher { +public: + void add(uint8_t byte) { m_state = (m_state ^ byte) * 0x100000001b3ULL; } + void add(std::span bytes) + { + for (auto byte : bytes) + add(byte); + } + void addNumber(uint64_t value) + { + for (unsigned shift = 0; shift < 64; shift += 8) + add(static_cast(value >> shift)); + } + uint64_t finish() const + { + uint64_t h = m_state; + h ^= h >> 33; + h *= 0xff51afd7ed558ccdULL; + h ^= h >> 33; + h *= 0xc4ceb9fe1a85ec53ULL; + h ^= h >> 33; + return std::min(h, std::numeric_limits::max() - 2); + } + +private: + uint64_t m_state { 0xcbf29ce484222325ULL }; +}; + +// Like OrderHashSet. OrderHasher::finish() never yields the two reserved values. +using OrderHashRankMap = UncheckedKeyHashMap, WTF::UnsignedWithZeroKeyHashTraits>; + +} // namespace + +std::optional orderFunctionKey(const UnlinkedFunctionExecutable& executable, const SourceCode& source) +{ + auto at = [](std::optional start, OrderFunctionKind kind) -> std::optional { + return start ? std::optional(OrderFunctionKey { *start, kind }) : std::nullopt; + }; + // Neither has a start of its own: `source` is a builtin's, and the whole scope the class is in. + if (executable.isBuiltinDefaultClassConstructor()) + return at(executable.classSourceStartWithoutMaterializing(), OrderFunctionKind::DefaultConstructor); + switch (executable.parseMode()) { + case SourceParseMode::ClassFieldInitializerMode: + return at(executable.firstClassElementOffsetWithoutMaterializing(), OrderFunctionKind::ClassFields); + case SourceParseMode::GeneratorBodyMode: + case SourceParseMode::AsyncFunctionBodyMode: + case SourceParseMode::AsyncArrowFunctionBodyMode: + case SourceParseMode::AsyncGeneratorBodyMode: + return at(source.startOffset(), OrderFunctionKind::InnerBody); + default: + return at(source.startOffset(), OrderFunctionKind::Function); + } +} + +RecordedOrderSource Decoder::orderSource() const +{ + return { m_payload, static_cast(m_cachedBytecode->entryOffset()) }; +} + +std::optional BytecodeOrderNames::nameOf(OrderFunctionKey key) const +{ + auto it = std::ranges::lower_bound(functions, key, { }, &Function::key); + if (it == functions.end() || it->key != key || !isValidOrderHash(it->name)) + return std::nullopt; + return it->name; +} + +// Over the UTF-16 code units, so it does not depend on which width the string happens to be stored in. +template +static uint64_t orderStringHash(std::span characters) +{ + OrderHasher hasher; + for (char16_t unit : characters) { + hasher.add(static_cast(unit)); + hasher.add(static_cast(unit >> 8)); + } + return hasher.finish(); +} + +uint64_t bytecodeOrderStringHash(const StringImpl& string) +{ + return string.is8Bit() ? orderStringHash(string.span8()) : orderStringHash(string.span16()); +} +#endif + static std::span inlineStringCharacters(std::span slot) { static_assert(std::endian::native == std::endian::little, "inline string slots are written as a little-endian word"); @@ -269,10 +360,37 @@ uint32_t EncoderStringTable::slotFor(const StringImpl& string) } // [u32 count][u32 offsets[count]][records: {u32 length|is8Bit<<31, u32 hash, chars, pad-to-4}...]; offsets are from the start of the blob. -Vector EncoderStringTable::serialize() const +Vector EncoderStringTable::serialize(std::span hotStringHashes) const { Vector out; uint32_t count = static_cast(m_strings.size()); + // The order the records are written in: ordinal order, unless an order file names strings to put first. + Vector order; +#if USE(BUN_JSC_ADDITIONS) + if (!hotStringHashes.empty()) { + OrderHashRankMap rankByHash; + for (uint32_t rank = 0; rank < hotStringHashes.size(); ++rank) { + if (isValidOrderHash(hotStringHashes[rank])) + rankByHash.add(hotStringHashes[rank], rank); + } + Vector> hot; // rank, ordinal + Vector rest; + for (uint32_t i = 0; i < count; ++i) { + auto it = rankByHash.find(bytecodeOrderStringHash(m_strings[i].get())); + if (it != rankByHash.end()) + hot.append({ it->value, i }); + else + rest.append(i); + } + std::ranges::sort(hot); + order.reserveInitialCapacity(count); + for (auto [rank, ordinal] : hot) + order.append(ordinal); + order.appendVector(rest); + } +#else + UNUSED_PARAM(hotStringHashes); +#endif size_t header = sizeof(uint32_t) * (1 + static_cast(count)); size_t body = 0; for (auto& s : m_strings) @@ -282,7 +400,8 @@ Vector EncoderStringTable::serialize() const uint32_t* words = std::bit_cast(out.mutableSpan().data()); words[0] = count; size_t offset = header; - for (uint32_t i = 0; i < count; ++i) { + for (uint32_t position = 0; position < count; ++position) { + uint32_t i = order.isEmpty() ? position : order[position]; words[1 + i] = static_cast(offset); const StringImpl& s = m_strings[i].get(); uint32_t* record = std::bit_cast(out.mutableSpan().data() + offset); @@ -349,6 +468,10 @@ Ref DecoderStringTable::createImpl(const Record& r) DecoderStringTable::Record DecoderStringTable::record(uint32_t ordinal) const { RELEASE_ASSERT(ordinal < m_count); +#if USE(BUN_JSC_ADDITIONS) + if (m_recorder) [[unlikely]] + m_recorder->didReadString(m_bytes, ordinal); +#endif const uint32_t* offsets = std::bit_cast(m_bytes.data() + sizeof(uint32_t)); size_t offset = offsets[ordinal]; RELEASE_ASSERT(!(offset % 4) && offset <= m_bytes.size() && m_bytes.size() - offset >= 2 * sizeof(uint32_t), offset, m_bytes.size()); @@ -631,6 +754,21 @@ String DecoderStringTable::stringFor(uint32_t ordinal) const return createImpl(record(ordinal)); } +#if USE(BUN_JSC_ADDITIONS) +void DecoderStringTable::enableFirstUseRecording(BytecodeOrderRecorder& recorder) +{ + m_recorder = &recorder; +} + +uint64_t DecoderStringTable::orderHashFor(uint32_t ordinal) const +{ + Record r = record(ordinal); + if (r.is8Bit) + return orderStringHash(std::span { r.characters, r.length }); + return orderStringHash(std::span { std::bit_cast(r.characters), r.length }); +} +#endif + template void DecoderStringTable::visitStrongReferences(Visitor& visitor, CollectionScope scope) { @@ -676,6 +814,9 @@ using SourceType = typename SourceTypeImpl::type; static constexpr size_t encoderMaxAlignment = 8; static constexpr size_t encoderMinPageSize = 4 * KB; +class CachedFunctionCodeBlock; +template class CachedWriteBarrier; + class Encoder { WTF_MAKE_NONCOPYABLE(Encoder); WTF_FORBID_HEAP_ALLOCATION; @@ -874,17 +1015,225 @@ class Encoder { // bodies follow breadth-first, and data that is only read on rare paths (expression info) goes after every body. // Decoding one block then reads one contiguous run of the payload rather than records scattered through every // descendant's subtree, so a mapped payload pages in only what is decoded. - void deferBody(Function&& encodeBody) { m_bodies.append(WTF::move(encodeBody)); } - void deferCold(Function&& encodeCold) { m_cold.append(WTF::move(encodeCold)); } + struct DeferredBody { + using Slot = CachedWriteBarrier; + // In the function's record, which stays where it is until release(). Null: the record has no such slot. + Slot* call { nullptr }; + Slot* construct { nullptr }; + WriteBarrier forCall; + WriteBarrier forConstruct; + // A link only: the code is written before the function's record is (placeLinkedFunction), which will point back at it. + const UnlinkedFunctionExecutable* writtenAheadOf { nullptr }; + + bool hasCode() const { return forCall || forConstruct; } + void encode(Encoder&) const; + bool pointBackAt(Encoder&, ptrdiff_t callRecord, ptrdiff_t constructRecord) const; + }; + void deferBody(const UnlinkedFunctionExecutable& executable, DeferredBody&& body) + { +#if USE(BUN_JSC_ADDITIONS) + if (m_link) { + deferLinkedBody(executable, WTF::move(body)); + return; + } +#else + UNUSED_PARAM(executable); +#endif + m_bodies.append(WTF::move(body)); + } + void deferCold(Function&& encodeCold) + { +#if USE(BUN_JSC_ADDITIONS) + if (m_link) { + m_link->coldData.append({ m_link->linkClass, m_link->rank, m_link->nextSequence++, WTF::move(encodeCold) }); + return; + } +#endif + m_cold.append(WTF::move(encodeCold)); + } void encodeDeferred() { while (!m_bodies.isEmpty()) - m_bodies.takeFirst()(); + m_bodies.takeFirst().encode(*this); while (!m_cold.isEmpty()) { m_cold.takeFirst()(); RELEASE_ASSERT(m_bodies.isEmpty()); } } + +#if USE(BUN_JSC_ADDITIONS) + // BytecodeLinkEncoder: bodies of every module of the link wait in one queue per region and are written region by region. + // In file order; a head is a module's cache entry, key, top-level code and its functions' records. + // The last region (BytecodeLinkRegions::ExpressionInfo) is what deferCold() defers. + enum class LinkClass : uint8_t { + EarlyHead = BytecodeLinkRegions::EarlyHeads, + Hot = BytecodeLinkRegions::Hot, + Unknown = BytecodeLinkRegions::Unknown, + LateHead = BytecodeLinkRegions::LateHeads, + Cold = BytecodeLinkRegions::Cold, + }; + static constexpr unsigned numberOfLinkClasses = BytecodeLinkRegions::ExpressionInfo; + struct LinkedBody { + uint64_t rank; // HOT: the order file's; otherwise module index, then source offset (source pre-order) + uint64_t sequence; + unsigned module; + DeferredBody body; + bool operator>(const LinkedBody& other) const { return rank != other.rank ? rank > other.rank : sequence > other.sequence; } + }; + // What deferCold() defers, with the place of the head or body it belongs to: the last region is written in the order + // of the regions before it, so that what a run reads of it (the position tables of the code that throws) is as + // close together as that code is. An array equal to an earlier one is shared, so it sits with its hottest user. + struct LinkedColdData { + LinkClass linkClass; + uint64_t rank; + uint64_t sequence; + Function encode; + bool operator<(const LinkedColdData& other) const { return std::tie(linkClass, rank, sequence) < std::tie(other.linkClass, other.rank, other.sequence); } + }; + struct LinkState { + WTF_DEPRECATED_MAKE_STRUCT_FAST_ALLOCATED(LinkState); + const BytecodeOrderNames* names { nullptr }; // of the module being added + // How many of the order file's hot functions are functions of this link: a file that fits nothing of the program + // should not go unnoticed. Nor should names that do not cover the code. + BitVector matchedHotRanks; + unsigned namedHotFunctions { 0 }; + unsigned placedHotFunctions { 0 }; + unsigned functionsWithoutName { 0 }; // of modules that have names + Vector coldData; + uint64_t rank { 0 }; // of the head (its module's index) or body being encoded + OrderHashRankMap hotRanks; + // The functions the recorded build had and its run did not decode. Empty: the order file does not say, and + // nothing is UNKNOWN. + OrderHashSet knownFunctions; + std::array, numberOfLinkClasses> queues; // min-heaps on (rank, sequence); heads have none + // Every function code block of every module, from the moment the module is added until the link ends: a collection + // may drop a function's code from a live executable (UnlinkedFunctionExecutable::codeBlockEdgeMayBeWeak), bodies + // are queued long after they were generated, and expression info is encoded last. + Vector> functionCodeBlocks; + uint64_t nextSequence { 0 }; + // Where each function that has code goes, and in what order there: decided when its module is added. + struct Placement { + LinkClass linkClass; + uint64_t rank; + // Code that is written before its function's record (DeferredBody::writtenAheadOf): whether it is already, + // and where its call and construct records are, 0 for none. + bool isWrittenAhead { false }; + ptrdiff_t callRecord { 0 }; + ptrdiff_t constructRecord { 0 }; + }; + UncheckedKeyHashMap placements; + // What is being encoded right now. + unsigned module { 0 }; + LinkClass linkClass { LinkClass::EarlyHead }; + LinkClass openRegion { LinkClass::EarlyHead }; // regions before it are complete + }; + void beginLink(std::span hotFunctions, std::span knownFunctions) + { + m_link = makeUnique(); + for (uint32_t rank = 0; rank < hotFunctions.size(); ++rank) { + if (isValidOrderHash(hotFunctions[rank])) + m_link->hotRanks.add(hotFunctions[rank], rank); + } + for (uint64_t hash : knownFunctions) { + if (isValidOrderHash(hash)) + m_link->knownFunctions.add(hash); + } + m_link->matchedHotRanks.ensureSize(hotFunctions.size()); + } + unsigned namedHotFunctions() const { return m_link->namedHotFunctions; } + unsigned placedHotFunctions() const { return m_link->placedHotFunctions; } + unsigned functionsWithoutName() const { return m_link->functionsWithoutName; } + // The names are the embedder's, and only for as long as its add* call lasts. + class NamesOfModuleBeingAdded { + WTF_MAKE_NONCOPYABLE(NamesOfModuleBeingAdded); + public: + NamesOfModuleBeingAdded(Encoder& encoder, const BytecodeOrderNames& names) + : m_encoder(encoder) + { + ASSERT(!encoder.m_link->names); + encoder.m_link->names = &names; + } + ~NamesOfModuleBeingAdded() { m_encoder.m_link->names = nullptr; } + private: + Encoder& m_encoder; + }; + // Top-level code of module `module` is about to be encoded. + void enterLinkedModule(unsigned module, bool isLate) + { + m_link->module = module; + m_link->linkClass = isLate ? LinkClass::LateHead : LinkClass::EarlyHead; + m_link->rank = module; + m_link->openRegion = m_link->linkClass; + } + // HOT if the recorded run ran it, UNKNOWN if the recorded build did not have it, else COLD; COLD whatever its name + // inside COLD code or a module the run did not evaluate (its twin ran, not it). Nothing for a duplicate. + std::optional placeLinkedFunction(const UnlinkedFunctionExecutable& executable, const SourceCode& source, unsigned module, LinkClass around, UnlinkedFunctionCodeBlock* forCall, UnlinkedFunctionCodeBlock* forConstruct) + { + LinkState& link = *m_link; + std::optional name; + if (auto key = orderFunctionKey(executable, source)) + name = link.names->nameOf(*key); + auto hot = name ? link.hotRanks.find(*name) : link.hotRanks.end(); + LinkState::Placement placement { LinkClass::Cold, static_cast(module) << 32 | static_cast(source.startOffset()) }; + if (around <= LinkClass::Unknown) { + if (hot != link.hotRanks.end()) + placement = { LinkClass::Hot, hot->value }; + else if (!link.knownFunctions.isEmpty() && !(name && link.knownFunctions.contains(*name))) + placement.linkClass = LinkClass::Unknown; + } + if (!link.placements.add(&executable, placement).isNewEntry) + return std::nullopt; + if (!name && link.names->hasAny()) + link.functionsWithoutName++; + if (hot != link.hotRanks.end() && !link.matchedHotRanks.quickSet(hot->value)) + link.namedHotFunctions++; + if (placement.linkClass == LinkClass::Hot) + link.placedHotFunctions++; + if (placement.linkClass == LinkClass::Hot && around == LinkClass::Unknown) { + DeferredBody body; + if (forCall) + body.forCall.setWithoutWriteBarrier(forCall); + if (forConstruct) + body.forConstruct.setWithoutWriteBarrier(forConstruct); + body.writtenAheadOf = &executable; + queueLinkedBody(placement, module, WTF::move(body)); + } + return placement.linkClass; + } + + void rootUntilLinkEnds(UnlinkedFunctionCodeBlock* codeBlock) + { + m_link->functionCodeBlocks.append(Strong(m_vm, codeBlock)); + } + void encodeLinkedBodies(LinkClass region) + { + m_link->openRegion = region; + auto& queue = m_link->queues[static_cast(region)]; + while (!queue.isEmpty()) { + std::ranges::pop_heap(queue, std::greater<> { }); + LinkedBody body = queue.takeLast(); + m_link->module = body.module; + m_link->linkClass = region; + m_link->rank = body.rank; + body.body.encode(*this); + } + } + bool hasQueuedLinkedBodies() const + { + return std::ranges::any_of(m_link->queues, [](auto& queue) { return !queue.isEmpty(); }); + } + void encodeLinkedCold() + { + RELEASE_ASSERT(!hasQueuedLinkedBodies() && m_cold.isEmpty()); + std::sort(m_link->coldData.begin(), m_link->coldData.end()); + for (auto& data : m_link->coldData) + data.encode(); + RELEASE_ASSERT(!hasQueuedLinkedBodies()); + m_link->coldData.clear(); + m_link->functionCodeBlocks.clear(); + } + void alignCurrentPageEnd() { m_currentPage->alignEnd(); } +#endif uint32_t nextStringOrdinal() { return m_numberStrings ? m_nextStringOrdinal++ : std::numeric_limits::max(); } // Content-sharing of arrays is only on while a code block encodes the arrays that may lie outside the block's own region. @@ -1012,6 +1361,56 @@ class Encoder { ptrdiff_t m_offset { 0 }; }; +#if USE(BUN_JSC_ADDITIONS) + void queueLinkedBody(LinkState::Placement placement, unsigned module, DeferredBody&& body) + { + LinkState& link = *m_link; + // A region that is complete takes nothing more: what should have gone there goes where it still can. + ASSERT(placement.linkClass >= link.openRegion); + if (placement.linkClass < link.openRegion) + placement.linkClass = link.openRegion == LinkClass::LateHead ? LinkClass::Cold : link.openRegion; + auto& queue = link.queues[static_cast(placement.linkClass)]; + queue.append(LinkedBody { placement.rank, link.nextSequence++, module, WTF::move(body) }); + std::ranges::push_heap(queue, std::greater<> { }); + } + + // The function's record was just written. + void deferLinkedBody(const UnlinkedFunctionExecutable& executable, DeferredBody&& body) + { + LinkState& link = *m_link; + ASSERT(!link.names); + // A function that had code when its module was added still has it (VM::keepsUnlinkedCode), and no other has. + auto placement = link.placements.find(&executable); + ASSERT(body.hasCode() == (placement != link.placements.end())); + if (!body.hasCode()) + return; + if (placement != link.placements.end() && placement->value.isWrittenAhead) { + if (body.pointBackAt(*this, placement->value.callRecord, placement->value.constructRecord)) + return; + placement = link.placements.end(); + } + if (placement == link.placements.end()) { + // Its code changed during the link, which it should not: keep what it has now, with the code nothing ran. + ASSERT_NOT_REACHED(); + for (auto* codeBlock : { body.forCall.get(), body.forConstruct.get() }) { + if (codeBlock) + rootUntilLinkEnds(codeBlock); + } + link.functionsWithoutName++; + queueLinkedBody({ LinkClass::Cold, static_cast(link.module) << 32 | std::numeric_limits::max() }, link.module, WTF::move(body)); + return; + } + queueLinkedBody(placement->value, link.module, WTF::move(body)); + } + void didWriteAhead(const UnlinkedFunctionExecutable& executable, ptrdiff_t callRecord, ptrdiff_t constructRecord) + { + auto& placement = m_link->placements.find(&executable)->value; + placement.isWrittenAhead = true; + placement.callRecord = callRecord; + placement.constructRecord = constructRecord; + } +#endif + void allocateNewPage(size_t size = 0) { static constexpr size_t minPageSize = encoderMinPageSize; @@ -1037,7 +1436,7 @@ class Encoder { UncheckedKeyHashMap m_ptrToOffsetMap; HashMap m_stringsByContents; // keyed by contents (StringHash), not identity LeafExecutableMap m_leafExecutables; - Deque> m_bodies; + Deque m_bodies; Deque> m_cold; uint32_t m_nextStringOrdinal { 0 }; EncoderStringTable* m_externalStrings; @@ -1046,6 +1445,9 @@ class Encoder { Vector m_sharedPrivateNameEnvironments; bool m_arraySharingEnabled { false }; UncheckedKeyHashMap, 1>, IntHash, WTF::UnsignedWithZeroKeyHashTraits> m_arraysByHash; +#if USE(BUN_JSC_ADDITIONS) + std::unique_ptr m_link; +#endif }; Decoder::Decoder(VM& vm, Ref cachedBytecode, RefPtr provider) @@ -1557,6 +1959,9 @@ class CachedWriteBarrier : public CachedObject> { m_ptr.encode(encoder, src.get()); } + // At a record that is already written, at `offset` in the payload. + void pointAt(Encoder& encoder, ptrdiff_t offset) { m_ptr.pointAtPayloadOffset(encoder, offset); } + void decode(Decoder& decoder, WriteBarrier& src, const JSCell* owner) const { Source* decodedPtr = m_ptr.decode(decoder); @@ -3363,6 +3768,8 @@ class CachedJSTextPosition : public CachedObject { return JSTextPosition { m_line, m_offset, m_lineStartOffset }; } + int offset() const { return m_offset; } + private: int m_line; int m_offset; @@ -3387,6 +3794,8 @@ class CachedClassElementDefinition : public CachedObject(m_kind); } + const CachedJSTextPosition& position() const { return m_position; } + private: CachedIdentifier m_ident; CachedJSTextPosition m_position; @@ -3466,20 +3875,13 @@ class CachedFunctionExecutableRareData : public CachedObject(this) + sizeof(uint32_t); - if (m_header & HasWrapperParameterNames) { + if (auto* p = tailField(HasWrapperParameterNames)) reinterpret_cast*>(p)->decode(decoder, rareData->m_generatorOrAsyncWrapperFunctionParameterNames); - p += sizeof(CachedVector); - } - if (m_header & HasClassElementDefinitions) { + if (auto* p = tailField(HasClassElementDefinitions)) reinterpret_cast*>(p)->decode(decoder, rareData->m_classElementDefinitions); - p += sizeof(CachedVector); - } - if (m_header & HasParentPrivateNameEnvironment) { + if (auto* p = tailField(HasParentPrivateNameEnvironment)) reinterpret_cast(p)->decode(decoder, rareData->m_parentPrivateNameEnvironment); - p += sizeof(CachedPrivateNameEnvironment); - } - if (m_header & HasClassSource) { + if (auto* p = tailField(HasClassSource)) { VarintReader reader(p); SourceCode& source = rareData->m_classSource; source.m_provider = decoder.provider(); @@ -3491,7 +3893,37 @@ class CachedFunctionExecutableRareData : public CachedObject classSourceStart() const + { + const uint8_t* p = tailField(HasClassSource); + return p ? std::optional(VarintReader(p).u32()) : std::nullopt; + } + std::optional firstClassElementOffset() const + { + const uint8_t* p = tailField(HasClassElementDefinitions); + auto definitions = p ? reinterpret_cast*>(p)->elements() : std::span { }; + return definitions.empty() ? std::nullopt : std::optional(definitions.front().position().offset()); + } + private: + // Where a member is in the tail, which has the members the header names, in the header's order but for the class + // source, which is last. Null: there is none. + const uint8_t* tailField(Header which) const + { + if (!(m_header & which)) + return nullptr; + const uint8_t* p = std::bit_cast(this) + sizeof(uint32_t); + for (auto [member, size] : { std::pair { HasWrapperParameterNames, sizeof(CachedVector) }, std::pair { HasClassElementDefinitions, sizeof(CachedVector) }, std::pair { HasParentPrivateNameEnvironment, sizeof(CachedPrivateNameEnvironment) }, std::pair { HasClassSource, size_t { 0 } } }) { + if (member == which) + return p; + if (m_header & member) + p += size; + } + RELEASE_ASSERT_NOT_REACHED(); + } + static VarintWriter packClassSource(const SourceCode& source) { VarintWriter writer; @@ -4082,8 +4514,11 @@ ALWAYS_INLINE void CachedCodeBlock::decode(Decoder& decoder, Unli uint16_t payloadIndex = decoder.persistentPayloadIndex(); uint32_t recordOffset = payloadIndex ? static_cast(decoder.offsetOf(this)) : 0; PersistentBytecodePayloads* payloads = payloadIndex ? &decoder.vm().persistentBytecodePayloads() : nullptr; - if (payloads) + if (payloads) { remembered = payloads->takeChildExecutables(payloadIndex, recordOffset); + if constexpr (std::is_same_v) + payloads->didDecodeFunctionBody(decoder.ptrForOffsetFromBase(0), recordOffset, layout.recordOffsetInRegion + sizeof(Record)); + } if (remembered.size() && remembered.size() == layout.functionDecls.count + layout.functionExprs.count) { auto decodeChildren = [&](const Array& array, auto& out, unsigned firstPosition, unsigned firstToDecode) { if (!array.count) @@ -4380,6 +4815,28 @@ auto CachedFunctionExecutable::view(ScalarsToView scalarsToView) const -> View return v; } +#if USE(BUN_JSC_ADDITIONS) +std::optional UnlinkedFunctionExecutable::classSourceStartWithoutMaterializing() const +{ + if (!m_membersAreDeferred) { + auto* rareData = m_members.live().rareData.get(); + return rareData && !rareData->m_classSource.isNull() ? std::optional(rareData->m_classSource.startOffset()) : std::nullopt; + } + auto* rareData = m_members.pending().record->slotsView().rareData; + return rareData ? (*rareData)->classSourceStart() : std::nullopt; +} + +std::optional UnlinkedFunctionExecutable::firstClassElementOffsetWithoutMaterializing() const +{ + if (!m_membersAreDeferred) { + auto* rareData = m_members.live().rareData.get(); + return rareData && !rareData->m_classElementDefinitions.isEmpty() ? std::optional(rareData->m_classElementDefinitions.first().position.offset) : std::nullopt; + } + auto* rareData = m_members.pending().record->slotsView().rareData; + return rareData ? (*rareData)->firstClassElementOffset() : std::nullopt; +} +#endif + void UnlinkedFunctionExecutable::materializeDeferredNameSlow() const { ASSERT(m_nameIsDeferred && m_membersAreDeferred); @@ -4497,14 +4954,41 @@ ALWAYS_INLINE void CachedFunctionExecutable::encode(Encoder& encoder, const Unli if (metadata && (!forCall || !forConstruct)) encoder.addLeafExecutable(&executable, encoder.offsetOf(this)); // CachedBytecode::addFunctionUpdate patches the Updatable layout's slots - encoder.deferBody([call, construct, &encoder, forCall, forConstruct] { - if (call) - call->encode(encoder, forCall); - if (construct) - construct->encode(encoder, forConstruct); - }); + encoder.deferBody(executable, { call, construct, forCall, forConstruct }); +} + +void Encoder::DeferredBody::encode(Encoder& encoder) const +{ +#if USE(BUN_JSC_ADDITIONS) + if (writtenAheadOf) { + auto write = [&](const WriteBarrier& codeBlock) -> ptrdiff_t { + return codeBlock ? encoder.offsetOf(CachedFunctionCodeBlock::create(encoder, *codeBlock.get())) : 0; + }; + ptrdiff_t callRecord = write(forCall); + encoder.didWriteAhead(*writtenAheadOf, callRecord, write(forConstruct)); + return; + } +#endif + if (call) + call->encode(encoder, forCall); + if (construct) + construct->encode(encoder, forConstruct); } +#if USE(BUN_JSC_ADDITIONS) +bool Encoder::DeferredBody::pointBackAt(Encoder& encoder, ptrdiff_t callRecord, ptrdiff_t constructRecord) const +{ + // What code the function has does not change during a link (VM::keepsUnlinkedCode). + if (!!call != !!callRecord || !!construct != !!constructRecord) + return false; + if (call) + call->pointAt(encoder, callRecord); + if (construct) + construct->pointAt(encoder, constructRecord); + return true; +} +#endif + ALWAYS_INLINE UnlinkedFunctionExecutable* CachedFunctionExecutable::decode(Decoder& decoder) const { UnlinkedFunctionExecutable* executable = new (NotNull, allocateCell(decoder.vm())) UnlinkedFunctionExecutable(decoder, *this); @@ -5038,15 +5522,40 @@ RefPtr encodeBuiltinFunction(VM& vm, const UnlinkedFunctionExecu return encoder.release(error, entry->payloadSizeSlot()); } -UnlinkedFunctionExecutable* decodeBuiltinFunction(VM& vm, Ref cachedBytecode, SourceProvider& provider, unsigned embedderStamp) +// Null if the payload cannot hold one where its entry is said to be. +template +static const Entry* cacheEntryOf(const CachedBytecode& cachedBytecode) +{ + auto span = cachedBytecode.span(); + size_t entryOffset = 0; +#if USE(BUN_JSC_ADDITIONS) + entryOffset = cachedBytecode.entryOffset(); +#endif + if (span.size() < entryOffset || span.size() - entryOffset < sizeof(Entry) || entryOffset % alignof(Entry)) + return nullptr; + return std::bit_cast(span.data() + entryOffset); +} + +static const GenericCacheEntry* cacheEntryOf(const CachedBytecode& cachedBytecode) { - if (cachedBytecode->span().size() < sizeof(BuiltinFunctionCacheEntry)) + return cacheEntryOf>(cachedBytecode); +} + +UnlinkedFunctionExecutable* decodeBuiltinFunction(VM& vm, Ref cachedBytecode, SourceProvider& provider, unsigned embedderStamp, Decoder::RecoverableCode recoverableCode) +{ + auto* entry = cacheEntryOf(cachedBytecode.get()); + if (!entry) return nullptr; unsigned sourceLength = provider.source().length(); - auto* entry = std::bit_cast(cachedBytecode->span().data()); - Ref decoder = Decoder::create(vm, WTF::move(cachedBytecode), &provider); + Ref decoder = Decoder::create(vm, WTF::move(cachedBytecode), &provider, recoverableCode); DeferGC deferGC(vm); - return entry->decode(decoder.get(), sourceLength, embedderStamp); + UnlinkedFunctionExecutable* executable = entry->decode(decoder.get(), sourceLength, embedderStamp); +#if USE(BUN_JSC_ADDITIONS) + // The builtin is the module; its code is recorded when it is run, like any function's. + if (auto* recorder = executable && decoder->canBorrowPayload() ? BytecodeOrderRecorder::ofVM(vm) : nullptr) [[unlikely]] + recorder->didDecodeModule(decoder->orderSource()); +#endif + return executable; } RefPtr encodeCodeBlock(VM& vm, const SourceCodeKey& key, const UnlinkedCodeBlock* codeBlock, FileSystem::FileHandle& fileHandle, BytecodeCacheError& error, EncoderStringTable* externalStrings, BytecodeCacheUpdatable updatable) @@ -5073,6 +5582,280 @@ RefPtr encodeCodeBlock(VM& vm, const SourceCodeKey& key, const U return encodeCodeBlock(vm, key, codeBlock, invalidFileHandle, error, externalStrings, updatable); } +#if USE(BUN_JSC_ADDITIONS) +WTF_MAKE_TZONE_ALLOCATED_IMPL(BytecodeLinkEncoder); + +struct BytecodeLinkEncoder::Impl { + WTF_DEPRECATED_MAKE_STRUCT_FAST_ALLOCATED(Impl); + + // A module's or program's code block under its key, or an embedder's builtin function (encodeBuiltinFunction). + struct Module { + SourceCodeKey key; + Strong root; + SourceCode source; + unsigned builtinEmbedderStamp { 0 }; + bool isLate { false }; + GenericCacheEntry* entry { nullptr }; // in the encoder's pages, which stay where they are until release() + uint32_t entryOffset { 0 }; + }; + bool isFinished { false }; + + Impl(VM& vm, EncoderStringTable* strings) + : vm(vm) + , encoder(vm, fileHandle, Encoder::NumberStrings::Yes, strings, BytecodeCacheUpdatable::No) + { + } + + void writeHead(unsigned index) + { + Module& module = modules[index]; + encoder.enterLinkedModule(index, module.isLate); + const ClassInfo* classInfo = module.root->classInfo(); + if (classInfo == UnlinkedProgramCodeBlock::info()) + module.entry = encodeCodeBlock(encoder, module.key, uncheckedDowncast(module.root.get())); + else if (classInfo == UnlinkedModuleProgramCodeBlock::info()) + module.entry = encodeCodeBlock(encoder, module.key, uncheckedDowncast(module.root.get())); + else { + RELEASE_ASSERT(classInfo == UnlinkedFunctionExecutable::info()); + auto* entry = encoder.template malloc(encoder); + entry->encode(encoder, *uncheckedDowncast(module.root.get()), static_cast(module.source.length()), module.builtinEmbedderStamp); + module.entry = entry; + } + module.entryOffset = safeCast(encoder.offsetOf(module.entry)); + } + + void add(Module&& module, const BytecodeOrderNames& names) + { + // finish() lets go of the modules' roots: what is known about their code is not good for a second link. + RELEASE_ASSERT(!isFinished); + module.isLate = isValidOrderHash(names.module) && notEvaluatedModules.contains(names.module); + unsigned index = modules.size(); + { + Encoder::NamesOfModuleBeingAdded namesOfModuleBeingAdded(encoder, names); + // `root` does not keep its functions' code alive (LinkState::functionCodeBlocks). + JSCell* root = module.root.get(); + auto head = module.isLate ? Encoder::LinkClass::LateHead : Encoder::LinkClass::EarlyHead; + if (root->classInfo() == UnlinkedFunctionExecutable::info()) + placeCodeOf(*uncheckedDowncast(root), index, module.source, head); + else + placeCodeOfFunctionsIn(*uncheckedDowncast(root), index, module.source, head); + } + modules.append(WTF::move(module)); + if (!modules[index].isLate) + writeHead(index); + } + + // Every function of the module that has code: keeps the code, and says where it goes. + void placeCodeOf(UnlinkedFunctionExecutable& executable, unsigned module, const SourceCode& around, Encoder::LinkClass aroundGoes) + { + auto [forCall, forConstruct] = executable.codeBlocksDecodingCached(vm); + if (!forCall && !forConstruct) + return; + SourceCode source = executable.linkedSourceCode(around); + auto goes = encoder.placeLinkedFunction(executable, source, module, aroundGoes, forCall, forConstruct); + if (!goes) + return; + for (UnlinkedFunctionCodeBlock* codeBlock : { forCall, forConstruct }) { + if (!codeBlock) + continue; + encoder.rootUntilLinkEnds(codeBlock); + placeCodeOfFunctionsIn(*codeBlock, module, source, *goes); + } + } + + void placeCodeOfFunctionsIn(UnlinkedCodeBlock& codeBlock, unsigned module, const SourceCode& source, Encoder::LinkClass goes) + { + for (unsigned i = 0; i < codeBlock.numberOfFunctionDecls(); ++i) + placeCodeOf(*codeBlock.functionDecl(i), module, source, goes); + for (unsigned i = 0; i < codeBlock.numberOfFunctionExprs(); ++i) + placeCodeOf(*codeBlock.functionExpr(i), module, source, goes); + } + + VM& vm; + FileSystem::FileHandle fileHandle; // invalid: the payload is built in memory + Encoder encoder; + Vector modules; + OrderHashSet notEvaluatedModules; +}; + +BytecodeLinkEncoder::BytecodeLinkEncoder(VM& vm, EncoderStringTable* strings, Hints&& hints) + : m_impl(makeUnique(vm, strings)) +{ + // Strings are numbered across the whole link, which only the shared table makes cheap for a module's Decoder. + RELEASE_ASSERT(strings); + // A function's record is written long after its module was added (with the body of the function around it, or when + // the link is finished) from what its executable holds then. + vm.keepUnlinkedCode(); + m_impl->encoder.beginLink(hints.hotFunctions.span(), hints.knownFunctions.span()); + for (uint64_t hash : hints.notEvaluatedModules) { + if (isValidOrderHash(hash)) + m_impl->notEvaluatedModules.add(hash); + } + // A module the run both evaluated and (as a duplicate) did not counts as evaluated. + for (uint64_t hash : hints.evaluatedModules) { + if (isValidOrderHash(hash)) + m_impl->notEvaluatedModules.remove(hash); + } +} + +VM& BytecodeLinkEncoder::vm() const { return m_impl->vm; } + +BytecodeLinkEncoder::~BytecodeLinkEncoder() +{ + m_impl->vm.stopKeepingUnlinkedCode(); +} + +void BytecodeLinkEncoder::addModule(const SourceCodeKey& key, UnlinkedCodeBlock* codeBlock, const SourceCode& source, const BytecodeOrderNames& names) +{ + m_impl->add(Impl::Module { key, Strong(m_impl->vm, codeBlock), source }, names); +} + +void BytecodeLinkEncoder::addBuiltinFunction(UnlinkedFunctionExecutable* executable, const SourceCode& source, unsigned embedderStamp, const BytecodeOrderNames& names) +{ + m_impl->add(Impl::Module { SourceCodeKey(), Strong(m_impl->vm, executable), source, embedderStamp }, names); +} + +auto BytecodeLinkEncoder::finish() -> Result +{ + static_assert(numberOfRegions == BytecodeLinkRegions::Count); + Result result; + Encoder& encoder = m_impl->encoder; + auto closeRegion = [&](unsigned region) { + result.regionEnds[region] = safeCast(encoder.currentOffset()); + }; + auto writeBodies = [&](Encoder::LinkClass region) { + encoder.encodeLinkedBodies(region); + closeRegion(static_cast(region)); + }; + closeRegion(BytecodeLinkRegions::EarlyHeads); + writeBodies(Encoder::LinkClass::Hot); + writeBodies(Encoder::LinkClass::Unknown); + for (unsigned index = 0; index < m_impl->modules.size(); ++index) { + if (m_impl->modules[index].isLate) + m_impl->writeHead(index); + } + closeRegion(BytecodeLinkRegions::LateHeads); + writeBodies(Encoder::LinkClass::Cold); + encoder.encodeLinkedCold(); + // Every module's entry records the size of the payload they share: a Decoder is made over all of it, so one whose + // span is shorter than that is a miss for every module, as for a payload of one module (GenericCacheEntry::isUpToDate). + encoder.alignCurrentPageEnd(); + uint32_t payloadSize = safeCast(encoder.currentOffset()); + for (auto& module : m_impl->modules) + *module.entry->payloadSizeSlot() = payloadSize; + BytecodeCacheError error; + result.payload = encoder.release(error); + RELEASE_ASSERT(result.payload && result.payload->size() == payloadSize && payloadSize <= static_cast(std::numeric_limits::max())); + result.regionEnds[BytecodeLinkRegions::ExpressionInfo] = safeCast(result.payload->size()); + result.namedHotFunctions = encoder.namedHotFunctions(); + result.placedHotFunctions = encoder.placedHotFunctions(); + result.functionsWithoutName = encoder.functionsWithoutName(); + for (auto& module : m_impl->modules) + result.entryOffsets.append(module.entryOffset); + m_impl->modules.clear(); + m_impl->isFinished = true; + return result; +} +#endif + +#if USE(BUN_JSC_ADDITIONS) +#endif + +#if USE(BUN_JSC_ADDITIONS) +static void digestCodeOf(VM&, OrderHasher&, CachedCodeDigest&, UnlinkedFunctionExecutable&); + +static void digestCodeBlock(VM& vm, OrderHasher& hasher, CachedCodeDigest& result, UnlinkedCodeBlock& codeBlock) +{ + auto addNumber = [&](uint64_t number) { + hasher.addNumber(number); + }; + result.codeBlocks++; + const auto& instructions = codeBlock.instructions(); + addNumber(instructions.sizeInBytes()); + hasher.add(std::span { static_cast(instructions.rawPointer()), instructions.sizeInBytes() }); + addNumber(codeBlock.constantRegisters().size()); + addNumber(codeBlock.numberOfIdentifiers()); + for (size_t i = 0; i < codeBlock.numberOfIdentifiers(); ++i) { + const Identifier& identifier = codeBlock.identifier(i); + addNumber(identifier.isNull() ? 0 : bytecodeOrderStringHash(*identifier.impl())); + } + addNumber(codeBlock.expressionInfo().byteSizeForGCPacing()); + unsigned declarations = codeBlock.numberOfFunctionDecls(); + unsigned expressions = codeBlock.numberOfFunctionExprs(); + addNumber(declarations); + addNumber(expressions); + for (unsigned i = 0; i < declarations + expressions; ++i) { + UnlinkedFunctionExecutable* executable = i < declarations ? codeBlock.functionDecl(i) : codeBlock.functionExpr(i - declarations); + digestCodeOf(vm, hasher, result, *executable); + } +} + +static void digestCodeOf(VM& vm, OrderHasher& hasher, CachedCodeDigest& result, UnlinkedFunctionExecutable& executable) +{ + auto [forCall, forConstruct] = executable.codeBlocksDecodingCached(vm); + hasher.add(static_cast(!!forCall | !!forConstruct << 1)); + if (forCall) + digestCodeBlock(vm, hasher, result, *forCall); + if (forConstruct) + digestCodeBlock(vm, hasher, result, *forConstruct); +} + +BytecodeOrderRecording bytecodeOrderRecording() +{ + BytecodeOrderRecording result; + UncheckedKeyHashMap, unsigned> sources; + // The first VM's first (a program's main thread), then each Worker's. + for (auto& recorder : BytecodeOrderRecorder::endRecordingInProcess()) { + auto recorded = recorder->take(); + Vector sourceOf(recorded.sources.size(), [&](size_t index) { + auto& source = recorded.sources[index]; + return sources.ensure({ source.payload, source.entryOffset }, [&] { + result.sources.append(source); + return static_cast(result.sources.size() - 1); + }).iterator->value; + }); + for (auto& function : recorded.functions) + result.functions.append({ sourceOf[function.source], function.key }); + for (unsigned module : recorded.modules) + result.evaluatedSources.append(sourceOf[module]); + for (unsigned module : recorded.rejectedModules) + result.rejectedSources.append(sourceOf[module]); + if (recorded.stringOrdinals.isEmpty()) + continue; + DecoderStringTable table(recorded.stringTable); // a reader of the same bytes that records nothing + for (uint32_t ordinal : recorded.stringOrdinals) + result.strings.append(table.orderHashFor(ordinal)); + } + return result; +} + +std::optional digestOfAllCachedBuiltinCode(VM& vm, const SourceCode& source, unsigned embedderStamp, Ref cachedBytecode) +{ + UnlinkedFunctionExecutable* executable = decodeBuiltinFunction(vm, WTF::move(cachedBytecode), *source.provider(), embedderStamp, Decoder::RecoverableCode::No); + if (!executable) + return std::nullopt; + OrderHasher hasher; + CachedCodeDigest result; + digestCodeOf(vm, hasher, result, *executable); + result.digest = hasher.finish(); + return result; +} + + +std::optional digestOfAllCachedCode(VM& vm, const SourceCode& source, bool isModule, Ref cachedBytecode) +{ + SourceCodeKey key = isModule ? sourceCodeKeyForSerializedModule(vm, source) : sourceCodeKeyForSerializedProgram(vm, source); + UnlinkedCodeBlock* codeBlock = decodeCodeBlockImpl(vm, key, WTF::move(cachedBytecode), Decoder::RecoverableCode::No); + if (!codeBlock) + return std::nullopt; + OrderHasher hasher; + CachedCodeDigest result; + digestCodeBlock(vm, hasher, result, *codeBlock); + result.digest = hasher.finish(); + return result; +} +#endif + RefPtr encodeFunctionCodeBlock(VM& vm, const UnlinkedFunctionCodeBlock* codeBlock, BytecodeCacheError& error) { FileSystem::FileHandle invalidFileHandle; @@ -5087,13 +5870,11 @@ RefPtr encodeFunctionCodeBlock(VM& vm, const UnlinkedFunctionCod std::optional decodeSourceCodeKey(VM& vm, Ref cachedBytecode) { - if (cachedBytecode->size() < sizeof(CacheEntry)) - return std::nullopt; - const auto* cachedEntry = std::bit_cast(cachedBytecode->span().data()); + const auto* cachedEntry = cacheEntryOf(cachedBytecode.get()); Ref decoder = Decoder::create(vm, WTF::move(cachedBytecode)); SourceCodeKey key; - if (!cachedEntry->decode(decoder.get(), key)) + if (!cachedEntry || !cachedEntry->decode(decoder.get(), key)) return std::nullopt; return key; } @@ -5101,12 +5882,28 @@ UnlinkedCodeBlock* decodeCodeBlockImpl(VM& vm, const SourceCodeKey& key, Refsize(); + bool payloadIsShared = false; +#if USE(BUN_JSC_ADDITIONS) + payloadIsShared = cachedBytecode->entryOffset(); +#endif if (Options::reportBytecodeCacheDecodeTimes()) [[unlikely]] before = MonotonicTime::now(); - if (cachedBytecodeSize < sizeof(CacheEntry)) +#if USE(BUN_JSC_ADDITIONS) + // A recording says which modules ran from their source although there was bytecode for them. + auto* recorder = cachedBytecode->payloadIsPersistent() ? BytecodeOrderRecorder::ofVM(vm) : nullptr; + RecordedOrderSource orderSource { cachedBytecode->span().data(), static_cast(cachedBytecode->entryOffset()) }; + auto rejected = [&]() -> UnlinkedCodeBlock* { + if (recorder) [[unlikely]] + recorder->didRejectModule(orderSource); return nullptr; - auto* cachedEntry = std::bit_cast(cachedBytecode->span().data()); + }; +#else + auto rejected = []() -> UnlinkedCodeBlock* { return nullptr; }; +#endif + auto* cachedEntry = cacheEntryOf(cachedBytecode.get()); + if (!cachedEntry) + return rejected(); // (A payload that turns out not to be for this key leaves nothing behind: its slot in VM::persistentBytecodePayloads() // goes with this Decoder and the code blocks it made.) Ref decoder = Decoder::create(vm, WTF::move(cachedBytecode), &key.source().provider(), recoverableCode); @@ -5114,14 +5911,19 @@ UnlinkedCodeBlock* decodeCodeBlockImpl(VM& vm, const SourceCodeKey& key, Refdecode(decoder.get(), entry)) - return nullptr; + return rejected(); } if (entry.first != key) - return nullptr; + return rejected(); + +#if USE(BUN_JSC_ADDITIONS) + if (recorder) [[unlikely]] + recorder->didDecodeModule(orderSource); +#endif if (Options::reportBytecodeCacheDecodeTimes()) [[unlikely]] { MonotonicTime after = MonotonicTime::now(); - dataLogLn("BytecodeCache: decoded ", key.source().provider().sourceURL(), " (", cachedBytecodeSize, " bytes) in ", (after - before).milliseconds(), " ms."); + dataLogLn("BytecodeCache: decoded ", key.source().provider().sourceURL(), " (", cachedBytecodeSize, payloadIsShared ? " bytes, shared with the other code of its link) in " : " bytes) in ", (after - before).milliseconds(), " ms."); } return entry.second; @@ -5129,10 +5931,9 @@ UnlinkedCodeBlock* decodeCodeBlockImpl(VM& vm, const SourceCodeKey& key, Ref cachedBytecode, const SourceCodeKey& key, SourceCodeType type) { - auto span = cachedBytecode->span(); - if (span.size() < sizeof(CacheEntry)) + auto* cachedEntry = cacheEntryOf(cachedBytecode.get()); + if (!cachedEntry) return false; - auto* cachedEntry = std::bit_cast(span.data()); Ref decoder = Decoder::create(vm, WTF::move(cachedBytecode)); return cachedEntry->isStillValid(decoder.get(), key, tagFromSourceCodeType(type)); } diff --git a/Source/JavaScriptCore/runtime/CachedTypes.h b/Source/JavaScriptCore/runtime/CachedTypes.h index 80f7c86347219..8ce8f52836223 100644 --- a/Source/JavaScriptCore/runtime/CachedTypes.h +++ b/Source/JavaScriptCore/runtime/CachedTypes.h @@ -33,15 +33,24 @@ #include "VariableEnvironment.h" #include #include +#include #include #include #include +#include +#include #include +#include +#include +#include +#include namespace JSC { class BytecodeCacheError; +class BytecodeOrderRecorder; class CachedBytecode; +class SourceCode; class SourceCodeKey; class SourceProvider; class CachedSymbolTable; @@ -91,7 +100,9 @@ class EncoderStringTable { // The 4-byte slot a cached non-symbol string occupies (CachedPtr's encoding): a 1-3 character Latin-1 string inline, // else an ordinal into this table, or the empty sentinel. DecoderStringTable::atomForSlot reads it back. JS_EXPORT_PRIVATE uint32_t slotFor(const StringImpl&); - JS_EXPORT_PRIVATE Vector serialize() const; + // `hotStringHashes` (bytecodeOrderStringHash values, hottest first; from a payload order file) moves those strings' + // records to the front, in that order; the rest follow in ordinal order. The offsets array stays indexed by ordinal. + JS_EXPORT_PRIVATE Vector serialize(std::span hotStringHashes = { }) const; static constexpr uint32_t maxOrdinal = (1u << 30) - 1; private: UncheckedKeyHashMap m_ordinals; @@ -151,6 +162,13 @@ class DecoderStringTable { void prefetchSlot(uint32_t ordinal) const; template void prefetchTarget(uint32_t ordinal) const; void prefetchLookup(AtomStringTable&, uint32_t ordinal) const; +#if USE(BUN_JSC_ADDITIONS) + // Payload order file recording: from now on tell the recorder each ordinal whose record is read. The table's bytes + // must outlive the recorder, that is the process. + JS_EXPORT_PRIVATE void enableFirstUseRecording(BytecodeOrderRecorder&); + // bytecodeOrderStringHash of the string, from its record. + uint64_t orderHashFor(uint32_t ordinal) const; +#endif private: static constexpr size_t recordHashOffset = sizeof(uint32_t); // EncoderStringTable::serialize's record layout struct Record { @@ -181,6 +199,9 @@ class DecoderStringTable { Vector m_cellOrdinals WTF_GUARDED_BY_LOCK(m_cellsLock); // the slots that hold a cell, for visitStrongReferences size_t m_visitedCount WTF_GUARDED_BY_LOCK(m_cellsLock) { 0 }; bool m_visitedThisCycle WTF_GUARDED_BY_LOCK(m_cellsLock) { false }; +#if USE(BUN_JSC_ADDITIONS) + RefPtr m_recorder; +#endif }; class VariableLengthObjectBase { @@ -222,6 +243,39 @@ class VariableLengthObjectBase { Offset m_offset; }; +#if USE(BUN_JSC_ADDITIONS) +// A payload order file is about code, and JSC does not name code: its embedder does, however it likes, and says which +// function a name is for by what JSC knows the function by. That is where it starts in its provider's text, in code units, +// and which of the functions that may start there it is. +enum class OrderFunctionKind : uint8_t { + Function, // starts where its parameters do (the start of the function's own SourceCode) + InnerBody, // what JSC makes of the body of an async function or of a generator; starts where that body does + ClassFields, // initializes a class's fields, and has no text but theirs; starts where the first of them does + DefaultConstructor, // of a class that does not write one: its text is a builtin's; starts where the class does +}; +struct OrderFunctionKey { + uint32_t start { 0 }; + OrderFunctionKind kind { OrderFunctionKind::Function }; + friend auto operator<=>(const OrderFunctionKey&, const OrderFunctionKey&) = default; +}; +// A table keyed by what an order file names something by reserves the top two values; names that come out of an order +// file or from the embedder are checked (isValidOrderHash) before they go into one. +using OrderHashSet = UncheckedKeyHashSet, WTF::UnsignedWithZeroKeyHashTraits>; +constexpr bool isValidOrderHash(uint64_t hash) { return hash <= std::numeric_limits::max() - 2; } + +// What a BytecodeOrderRecorder saw decoded. It knows a source by the payload its code came out of, and where in it its +// cache entry starts: a payload that is recorded outlives the program, so its address says which it is, for good. +struct RecordedOrderSource { + const uint8_t* payload { nullptr }; + uint32_t entryOffset { 0 }; // CachedBytecode::entryOffset + friend bool operator==(const RecordedOrderSource&, const RecordedOrderSource&) = default; +}; +struct RecordedOrderFunction { + unsigned source; // an index into the list of sources it comes with + OrderFunctionKey key; +}; +#endif + class Decoder : public RefCounted { WTF_MAKE_NONCOPYABLE(Decoder); @@ -267,6 +321,9 @@ class Decoder : public RefCounted { // This decoder's payload in VM::persistentBytecodePayloads(), or 0: what a code block decoded from it needs to remember // (with its record's offset) to be decoded again later. uint16_t persistentPayloadIndex() const { return m_persistentPayloadIndex; } +#if USE(BUN_JSC_ADDITIONS) + RecordedOrderSource orderSource() const; +#endif void clearPersistentPayloadIndex() { m_persistentPayloadIndex = 0; } void addRetainedTableSizes(size_t& mappedPointers, size_t& atomsByOrdinal, size_t& finalizers) const { @@ -303,13 +360,103 @@ class Decoder : public RefCounted { JS_EXPORT_PRIVATE RefPtr encodeCodeBlock(VM&, const SourceCodeKey&, const UnlinkedCodeBlock*, EncoderStringTable* = nullptr, BytecodeCacheUpdatable = BytecodeCacheUpdatable::Yes); JS_EXPORT_PRIVATE RefPtr encodeCodeBlock(VM&, const SourceCodeKey&, const UnlinkedCodeBlock*, FileSystem::FileHandle&, BytecodeCacheError&, EncoderStringTable* = nullptr, BytecodeCacheUpdatable = BytecodeCacheUpdatable::Yes); +#if USE(BUN_JSC_ADDITIONS) +// What an order file names a string of a payload by. +JS_EXPORT_PRIVATE uint64_t bytecodeOrderStringHash(const StringImpl&); +// The SourceCode is the function's own (UnlinkedFunctionExecutable::linkedSourceCode). Nothing for a function there is +// no telling the start of (it has no name then). +JS_EXPORT_PRIVATE std::optional orderFunctionKey(const UnlinkedFunctionExecutable&, const SourceCode&); + +// The embedder's names for the code of one source text. +struct BytecodeOrderNames { + struct Function { + OrderFunctionKey key; + uint64_t name; + }; + uint64_t module { std::numeric_limits::max() }; // the top-level code's; not a name (isValidOrderHash) by default + std::span functions; // sorted by key; the embedder's, for the length of the add* call + bool hasAny() const { return !functions.empty() || isValidOrderHash(module); } + std::optional nameOf(OrderFunctionKey) const; +}; + +// What every VM of the process, alive or gone, read out of its persistent payloads: each list in first-use order, the +// first VM's first (what two VMs both used is listed for each). The embedder makes an order file of it, in its names. +struct BytecodeOrderRecording { + Vector sources; + Vector functions; // decoded to be run + Vector evaluatedSources; // modules and builtins decoded: indices into `sources` + Vector rejectedSources; // modules whose bytecode was not for their source, in a VM: they ran from source there + Vector strings; // bytecodeOrderStringHash of each string read +}; +// Taking it ends the recording, for the process. +JS_EXPORT_PRIVATE BytecodeOrderRecording bytecodeOrderRecording(); + +// For checking one payload layout against another: decodes ALL the code `cachedBytecode` holds for `source` (every +// function, however deeply nested, and each block's expression info) and digests, in tree order, each block's +// instructions, constant count, identifiers and expression info size. Nullopt if the payload is not for `source`. +// On a VM that records (BytecodeOrderRecorder) the module and every string count as used: digest after +// bytecodeOrderRecording(). +struct CachedCodeDigest { + uint64_t digest { 0 }; + unsigned codeBlocks { 0 }; +}; +JS_EXPORT_PRIVATE std::optional digestOfAllCachedCode(VM&, const SourceCode&, bool isModule, Ref); +JS_EXPORT_PRIVATE std::optional digestOfAllCachedBuiltinCode(VM&, const SourceCode&, unsigned embedderStamp, Ref); + +// `bun build --compile --bytecode` with a payload order file: every module of the link is encoded into ONE payload, laid +// out by how the recorded run used it. Regions, in file order, each written to completion before the next starts: +// 0 heads (cache entry, key, top-level code, its functions' records) of modules the run evaluated, or did not know +// 1 HOT bodies, in the order file's order 2 UNKNOWN bodies: functions the recorded build did not have +// 3 heads of modules the run knew and did not evaluate 4 all other bodies, in source order 5 expression info. +// So every offset is final when it is written: a reference to something earlier is a plain delta, and a function +// record's body slots and a code block's expression-info slot are filled in when their target is written, as in a +// single-module payload. Every module's unlinked code stays alive until finish(). +class BytecodeLinkEncoder { + WTF_MAKE_NONCOPYABLE(BytecodeLinkEncoder); + WTF_MAKE_TZONE_ALLOCATED_EXPORT(BytecodeLinkEncoder, JS_EXPORT_PRIVATE); +public: + // What an order file says, in the names the embedder gives code (BytecodeOrderNames). + struct Hints { + Vector hotFunctions; // in first-decode order + Vector knownFunctions; // the other functions the recorded build had; empty = not recorded + Vector evaluatedModules; + Vector notEvaluatedModules; + }; + static constexpr unsigned numberOfRegions = 6; // BytecodeLinkRegions::Count + struct Result { + RefPtr payload; + Vector entryOffsets; // per addModule call, in call order + unsigned namedHotFunctions { 0 }; // of Hints::hotFunctions, how many name a function of this link + unsigned placedHotFunctions { 0 }; // functions of this link that went to HOT + unsigned functionsWithoutName { 0 }; // functions with code that the names of their module, which has some, do not cover + std::array regionEnds { }; + }; + + // The shared string table is required. Destroy it, as it is used, on its VM's thread with the VM's lock held. + JS_EXPORT_PRIVATE BytecodeLinkEncoder(VM&, EncoderStringTable*, Hints&&); + JS_EXPORT_PRIVATE ~BytecodeLinkEncoder(); + // `source` is the whole module, as given to the parser; the code block is a module's or a program's. + JS_EXPORT_PRIVATE void addModule(const SourceCodeKey&, UnlinkedCodeBlock*, const SourceCode&, const BytecodeOrderNames&); + // An embedder's builtin (what encodeBuiltinFunction takes), `source` being all of its source: decodeBuiltinFunction + // reads it back given the payload and the entry's offset. The builtin is a module, and a function of it. + JS_EXPORT_PRIVATE void addBuiltinFunction(UnlinkedFunctionExecutable*, const SourceCode& source, unsigned embedderStamp, const BytecodeOrderNames&); + JS_EXPORT_PRIVATE Result finish(); + JS_EXPORT_PRIVATE VM& vm() const; + +private: + struct Impl; + std::unique_ptr m_impl; +}; +#endif + UnlinkedCodeBlock* decodeCodeBlockImpl(VM&, const SourceCodeKey&, Ref, Decoder::RecoverableCode = Decoder::RecoverableCode::Yes); // An embedder's JS builtin (a root UnlinkedFunctionExecutable from BuiltinExecutables::createExecutable), with its code // blocks generated recursively beforehand (see recursivelyGenerateUnlinkedCodeBlocksForFunction). `embedderStamp` // identifies the builtin source's contents; decode checks it and the source length instead of hashing the source. JS_EXPORT_PRIVATE RefPtr encodeBuiltinFunction(VM&, const UnlinkedFunctionExecutable*, unsigned sourceLength, unsigned embedderStamp, EncoderStringTable* = nullptr, BytecodeCacheUpdatable = BytecodeCacheUpdatable::Yes); -JS_EXPORT_PRIVATE UnlinkedFunctionExecutable* decodeBuiltinFunction(VM&, Ref, SourceProvider&, unsigned embedderStamp); +// RecoverableCode::No: for a pass over everything a payload holds, which must not share code with the program (see Decoder::create). +JS_EXPORT_PRIVATE UnlinkedFunctionExecutable* decodeBuiltinFunction(VM&, Ref, SourceProvider&, unsigned embedderStamp, Decoder::RecoverableCode = Decoder::RecoverableCode::Yes); template UnlinkedCodeBlockType* decodeCodeBlock(VM& vm, const SourceCodeKey& key, Ref cachedBytecode, Decoder::RecoverableCode recoverableCode = Decoder::RecoverableCode::Yes) diff --git a/Source/JavaScriptCore/runtime/ModuleProgramExecutable.cpp b/Source/JavaScriptCore/runtime/ModuleProgramExecutable.cpp index 592e059ed45ab..49ec2a78009f7 100644 --- a/Source/JavaScriptCore/runtime/ModuleProgramExecutable.cpp +++ b/Source/JavaScriptCore/runtime/ModuleProgramExecutable.cpp @@ -179,8 +179,7 @@ ModuleProgramExecutable* ModuleProgramExecutable::tryCreate(JSGlobalObject* glob ModuleProgramExecutable* executable = new (NotNull, allocateCell(vm)) ModuleProgramExecutable(globalObject, source, linker, moduleScopeSymbolTables); executable->finishCreation(vm); - if (!executable->getUnlinkedCodeBlock(globalObject)) [[unlikely]] // This generates and binds unlinked code block. - return nullptr; + executable->getUnlinkedCodeBlock(globalObject); // This generates and binds unlinked code block. Null: it has thrown. RETURN_IF_EXCEPTION(scope, nullptr); return executable; } diff --git a/Source/JavaScriptCore/runtime/ScriptExecutable.cpp b/Source/JavaScriptCore/runtime/ScriptExecutable.cpp index 0ec6a0aecb83e..d65e5dc6fee10 100644 --- a/Source/JavaScriptCore/runtime/ScriptExecutable.cpp +++ b/Source/JavaScriptCore/runtime/ScriptExecutable.cpp @@ -79,8 +79,11 @@ void ScriptExecutable::clearCode(IsoCellSet& clearableCodeSet, ClearCode mode) m_jitCodeForCallWithArityCheck = CodePtr(); m_jitCodeForConstructWithArityCheck = CodePtr(); + bool keepsUnlinkedCode = vm().keepsUnlinkedCode(); auto clearGlobalCode = [&](GlobalExecutable* executable, bool canDecodeAgain) { executable->m_codeBlock.clear(); + if (keepsUnlinkedCode) [[unlikely]] + return; UnlinkedCodeBlock* unlinkedCodeBlock = executable->m_unlinkedCodeBlock.get(); if (mode == ClearCode::All || (canDecodeAgain && unlinkedCodeBlock && unlinkedCodeBlock->cachedPayloadIndex())) executable->m_unlinkedCodeBlock.clear(); @@ -110,7 +113,8 @@ void ScriptExecutable::clearCode(IsoCellSet& clearableCodeSet, ClearCode mode) executable->m_hasReleasedUnlinkedCode = false; } else { executable->m_codeBlock.clear(); - executable->releaseUnlinkedCodeIfRecoverable(vm()); + if (!keepsUnlinkedCode) + executable->releaseUnlinkedCodeIfRecoverable(vm()); } break; } diff --git a/Source/JavaScriptCore/runtime/VM.cpp b/Source/JavaScriptCore/runtime/VM.cpp index 89386c2c0117f..93dd39de99c10 100644 --- a/Source/JavaScriptCore/runtime/VM.cpp +++ b/Source/JavaScriptCore/runtime/VM.cpp @@ -1126,7 +1126,10 @@ void VM::deleteAllRegExpCode() void VM::deleteAllCode(DeleteAllCodeEffort effort) { whenIdle([=, this] () { - m_codeCache->clear(); + if (keepsUnlinkedCode()) + m_codeCache->write(); // what clear() does first + else + m_codeCache->clear(); m_builtinExecutables->clear(); deleteAllRegExpCode(); heap.deleteAllCodeBlocks(effort); @@ -1136,6 +1139,14 @@ void VM::deleteAllCode(DeleteAllCodeEffort effort) }); } +void VM::deleteAllCodeToGenerateItAgain(DeleteAllCodeEffort effort) +{ + whenIdle([=, this] () { + SetForScope generatingAgain(m_isDeletingAllCodeToGenerateItAgain, true); + deleteAllCode(effort); // runs now: the VM is idle + }); +} + bool VM::shrinkFootprintNow(OptionSet mode) { // Not under JS, and not from inside the collector (a finalizer, a heap observer): deleting code waits for a @@ -1162,7 +1173,8 @@ bool VM::shrinkFootprintNow(OptionSet mode) if (keepCodeInUse) unlinkedCode.add(UnlinkedCodeToDelete::OnlyWithoutLinkedCode); heap.deleteAllUnlinkedCodeBlocks(PreventCollectionAndDeleteAllCode, unlinkedCode); - m_codeCache->clearCodeDecodedFromPersistentPayloads(); + if (!keepsUnlinkedCode()) + m_codeCache->clearCodeDecodedFromPersistentPayloads(); if (!keepCodeInUse) deleteAllRegExpCode(); else if (Options::releaseIdleRegExpCodeWhenShrinkingFootprint() && !numberOfActiveJITPlans()) { diff --git a/Source/JavaScriptCore/runtime/VM.h b/Source/JavaScriptCore/runtime/VM.h index caa9ccfad96c6..71079e295c7da 100644 --- a/Source/JavaScriptCore/runtime/VM.h +++ b/Source/JavaScriptCore/runtime/VM.h @@ -1025,6 +1025,34 @@ class VM : public ThreadSafeRefCountedWithSuppressingSaferCPPChecking { PersistentBytecodePayloads& persistentBytecodePayloads(); PersistentBytecodePayloads* persistentBytecodePayloadsIfExists() { return m_persistentBytecodePayloads.get(); } +#if USE(BUN_JSC_ADDITIONS) + // While anybody asks, deleteAllCode(), shrinkFootprintNow() and whoever else goes through + // Heap::deleteAllUnlinkedCodeBlocks or ScriptExecutable::clearCode leave unlinked code where it is: executables + // keep their code blocks, code decoded from a bytecode cache is not returned to it, a program or module keeps its + // top-level code, and the code cache is not emptied. Linked code is dropped as ever. (Not covered, and not needed: + // the code cache evicts by size and age, and a collection drops aged code that nothing roots.) + // Who asks: a BytecodeLinkEncoder, which writes a function's record long after its module was added from what the + // executable holds then, so that an emptied executable would silently leave the payload without the body; and a + // run that records what it decodes (PersistentBytecodePayloads::enableOrderRecording). Neither is a program's + // steady state. + // deleteAllCodeToGenerateItAgain() is not held back: see there. + void keepUnlinkedCode() { ++m_unlinkedCodeKeepers; } + void stopKeepingUnlinkedCode() + { + RELEASE_ASSERT(m_unlinkedCodeKeepers); + --m_unlinkedCodeKeepers; + } + // A run that records keeps it until its recording is taken, which any thread may do (BytecodeOrderRecorder::take). + void keepUnlinkedCodeUntil(const std::atomic& isOver) { m_unlinkedCodeIsKeptUntil = &isOver; } + bool keepsUnlinkedCode() const + { + bool isKept = m_unlinkedCodeKeepers || (m_unlinkedCodeIsKeptUntil && !m_unlinkedCodeIsKeptUntil->load()); + return isKept && !m_isDeletingAllCodeToGenerateItAgain; + } +#else + bool keepsUnlinkedCode() const { return false; } +#endif + // See LazyCallLinkInfo. CallSiteData* neverExecutedCallSiteData() { return m_neverExecutedCallSiteData; } CallSiteData* executedOnceCallSiteData() { return m_executedOnceCallSiteData; } @@ -1044,6 +1072,10 @@ class VM : public ThreadSafeRefCountedWithSuppressingSaferCPPChecking { JS_EXPORT_PRIVATE void setStartupJITDeferralScale(double); // <= 1 ends the window JS_EXPORT_PRIVATE void deleteAllCode(DeleteAllCodeEffort); + // For code that has to be generated differently from now on (a debugger attached, a profiler turned on): + // functions that kept their unlinked code would go on running without the hooks, so keepsUnlinkedCode() does not + // hold this back. A recording made across it is less exact (code is decoded again); a link has no debugger. + JS_EXPORT_PRIVATE void deleteAllCodeToGenerateItAgain(DeleteAllCodeEffort); JS_EXPORT_PRIVATE void deleteAllLinkedCode(DeleteAllCodeEffort); void deleteAllRegExpCode(); @@ -1370,6 +1402,11 @@ class VM : public ThreadSafeRefCountedWithSuppressingSaferCPPChecking { HeapAnalyzer* m_activeHeapAnalyzer { nullptr }; std::unique_ptr m_codeCache; std::unique_ptr m_persistentBytecodePayloads; +#if USE(BUN_JSC_ADDITIONS) + unsigned m_unlinkedCodeKeepers { 0 }; // the VM's thread only + const std::atomic* m_unlinkedCodeIsKeptUntil { nullptr }; // a BytecodeOrderRecorder's, which the process keeps for good +#endif + bool m_isDeletingAllCodeToGenerateItAgain { false }; CallSiteData* m_neverExecutedCallSiteData { nullptr }; CallSiteData* m_executedOnceCallSiteData { nullptr }; CallSiteData* m_notExecutedTailCallSiteData { nullptr };