[JSC] Bytecode cache: lay the modules of a link out in one payload by an order file

Source/JavaScriptCore/bytecode/UnlinkedFunctionExecutable.cpp

@@ -295,6 +295,15 @@ UnlinkedFunctionCodeBlock* UnlinkedFunctionExecutable::unlinkedCodeBlockFor(
     return result;
 }
 
+#if USE(BUN_JSC_ADDITIONS)
+std::pair<UnlinkedFunctionCodeBlock*, UnlinkedFunctionCodeBlock*> UnlinkedFunctionExecutable::codeBlocksDecodingCached(VM& vm)
+{
+    if (m_isCached)
+        decodeCachedCodeBlocks(vm);
+    return { m_unlinkedCodeBlockForCall.get(), m_unlinkedCodeBlockForConstruct.get() };
+}
+#endif
+
 void UnlinkedFunctionExecutable::decodeCachedCodeBlocks(VM& vm)
 {
     ASSERT(m_isCached);

Source/JavaScriptCore/bytecode/UnlinkedFunctionExecutable.h

@@ -213,6 +213,11 @@ class UnlinkedFunctionExecutable final : public JSCell {
     bool isBuiltinFunction() const { return m_isBuiltinFunction; }
     // The code blocks are still (or again) in the bytecode cache this was decoded from.
     bool isCached() const { return m_isCached; }
+#if USE(BUN_JSC_ADDITIONS)
+    // The code blocks for call and construct as they are once whatever the bytecode cache holds for this function is
+    // decoded; never generates one (unlinkedCodeBlockFor does).
+    JS_EXPORT_PRIVATE std::pair<UnlinkedFunctionCodeBlock*, UnlinkedFunctionCodeBlock*> codeBlocksDecodingCached(VM&);
+#endif
     ConstructAbility constructAbility() const { return static_cast<ConstructAbility>(m_constructAbility); }
     JSParserScriptMode scriptMode() const { return static_cast<JSParserScriptMode>(m_scriptMode); }
     bool isClassConstructorFunction() const

Source/JavaScriptCore/runtime/CachedBytecode.cpp

@@ -32,6 +32,7 @@
 #include "UnlinkedCodeBlock.h"
 #include "UnlinkedFunctionExecutable.h"
 #include "WeakInlines.h"
+#include <wtf/NeverDestroyed.h>
 #include <wtf/TZoneMallocInlines.h>
 
 WTF_ALLOW_UNSAFE_BUFFER_USAGE_BEGIN
@@ -40,6 +41,99 @@ namespace JSC {
 
 WTF_MAKE_TZONE_ALLOCATED_IMPL(PersistentBytecodePayloads);
 
+#if USE(BUN_JSC_ADDITIONS)
+WTF_MAKE_TZONE_ALLOCATED_IMPL(BytecodeOrderRecorder);
+
+BytecodeOrderRecorder::BytecodeOrderRecorder() = default;
+BytecodeOrderRecorder::~BytecodeOrderRecorder() = default;
+
+static Lock s_orderRecordersLock;
+static Vector<Ref<BytecodeOrderRecorder>>& orderRecorders() WTF_REQUIRES_LOCK(s_orderRecordersLock)
+{
+    static NeverDestroyed<Vector<Ref<BytecodeOrderRecorder>>> recorders;
+    return recorders;
+}
+
+Ref<BytecodeOrderRecorder> BytecodeOrderRecorder::create()
+{
+    Ref recorder = adoptRef(*new BytecodeOrderRecorder);
+    Locker locker { s_orderRecordersLock };
+    orderRecorders().append(recorder);
+    return recorder;
+}
+
+Vector<Ref<BytecodeOrderRecorder>> BytecodeOrderRecorder::allInProcess()
+{
+    Locker locker { s_orderRecordersLock };
+    return orderRecorders();
+}
+
+BytecodeOrderRecorder* BytecodeOrderRecorder::ofVM(VM& vm)
+{
+    auto* payloads = vm.persistentBytecodePayloadsIfExists();
+    return payloads ? payloads->orderRecorder() : nullptr;
+}
+
+BytecodeOrderRecorder* BytecodeOrderRecorder::ifRecording(VM& vm)
+{
+    auto* recorder = ofVM(vm);
+    return recorder && !recorder->m_pauseDepth ? recorder : nullptr;
+}
+
+void BytecodeOrderRecorder::didDecodeFunction(const void* record, UnlinkedFunctionCodeBlock& codeBlock)
+{
+    if (m_recordsOfCodeBlocks) {
+        m_recordsOfCodeBlocks->set(&codeBlock, record);
+        return;
+    }
+    if (m_pauseDepth)
+        return;
+    Locker locker { m_lock };
+    if (m_seenFunctions.add(record).isNewEntry)
+        m_recorded.functions.append(record);
+}
+
+void BytecodeOrderRecorder::didDecodeModule(const void* record)
+{
+    Locker locker { m_lock };
+    if (m_seenModules.add(record).isNewEntry)
+        m_recorded.modules.append(record);
+}
+
+void BytecodeOrderRecorder::didReadString(std::span<const uint8_t> stringTable, uint32_t ordinal)
+{
+    if (m_pauseDepth)
+        return;
+    Locker locker { m_lock };
+    if (m_seenStrings.set(ordinal))
+        return;
+    // Ordinals mean something in one table only: a VM has one for as long as it records.
+    RELEASE_ASSERT(m_recorded.stringOrdinals.isEmpty() || m_recorded.stringTable.data() == stringTable.data());
+    m_recorded.stringTable = stringTable;
+    m_recorded.stringOrdinals.append(ordinal);
+}
+
+auto BytecodeOrderRecorder::snapshot() const -> Snapshot
+{
+    Locker locker { m_lock };
+    return m_recorded;
+}
+
+void PersistentBytecodePayloads::setLinkedPayload(std::span<const uint8_t> payload, const std::array<uint32_t, BytecodeLinkRegions::Count>& regionEnds)
+{
+    m_linkedPayloadBase = payload.data();
+    m_linkedPayloadStatistics = { };
+    m_linkedPayloadStatistics.regionEnds = regionEnds;
+}
+
+BytecodeOrderRecorder& PersistentBytecodePayloads::enableOrderRecording()
+{
+    if (!m_orderRecorder)
+        m_orderRecorder = BytecodeOrderRecorder::create();
+    return *m_orderRecorder;
+}
+#endif
+
 PersistentBytecodePayloads::PersistentBytecodePayloads(VM& vm)
     : m_vm(vm)
 {

Source/JavaScriptCore/runtime/CachedBytecode.h

@@ -30,19 +30,25 @@
 #include "ParserModes.h"
 #include "Weak.h"
 #include "WeakGCHashTable.h"
-#include <wtf/HashMap.h>
+#include <array>
+#include <wtf/BitVector.h>
 #include <wtf/FixedVector.h>
+#include <wtf/HashMap.h>
+#include <wtf/HashSet.h>
+#include <wtf/Lock.h>
 #include <wtf/MallocSpan.h>
 #include <wtf/Noncopyable.h>
 #include <wtf/RefCounted.h>
 #include <wtf/TZoneMalloc.h>
+#include <wtf/ThreadSafeRefCounted.h>
 #include <wtf/Vector.h>
 
 namespace JSC {
 
 class Decoder;
 class SourceProvider;
 class UnlinkedCodeBlock;
+class UnlinkedFunctionCodeBlock;
 class UnlinkedFunctionExecutable;
 class VM;
 
@@ -84,6 +90,12 @@ class CachedBytecode : public RefCounted<CachedBytecode> {
     // Where the root record starts within the payload (a function code block is written after its own arrays).
     size_t rootOffset() const { return m_rootOffset; }
     void setRootOffset(size_t offset) { m_rootOffset = offset; }
+#if USE(BUN_JSC_ADDITIONS)
+    // Where this code's cache entry starts within the payload. Non-zero when several modules were encoded into one
+    // payload (BytecodeLinkEncoder): every offset a Decoder keeps stays relative to the start of the shared payload.
+    size_t entryOffset() const { return m_entryOffset; }
+    void setEntryOffset(size_t offset) { m_entryOffset = offset; }
+#endif
     void setPayloadIsPersistent() { m_payload.setIsPersistent(); }
     bool payloadIsOwnedOrPersistent() const { return m_payload.isOwnedOrPersistent(); }
     bool hasUpdates() const { return !m_updates.isEmpty(); }
@@ -102,10 +114,92 @@ class CachedBytecode : public RefCounted<CachedBytecode> {
     size_t m_size { 0 };
     CachePayload m_payload;
     size_t m_rootOffset { 0 };
+#if USE(BUN_JSC_ADDITIONS)
+    size_t m_entryOffset { 0 };
+#endif
     LeafExecutableMap m_leafExecutables;
     Vector<CacheUpdate> m_updates;
 };
 
+#if USE(BUN_JSC_ADDITIONS)
+// The regions of a payload written by BytecodeLinkEncoder, in the order they lie in it.
+namespace BytecodeLinkRegions {
+enum : unsigned { EarlyHeads, Hot, Unknown, LateHeads, Cold, ExpressionInfo, Count };
+}
+
+// What one VM read out of its persistent payloads, in first-use order: the input of a payload order file. Exists only
+// while recording (PersistentBytecodePayloads::enableOrderRecording). Every recorder of the process stays registered,
+// whether or not its VM is still alive, and any thread may take a snapshot of it (the thread that writes the order file
+// when the process exits is not the thread of a Worker's VM), so what it remembers must not die with the VM: functions
+// and modules are hashes, and strings are ordinals into a table whose bytes the embedder keeps for good.
+class BytecodeOrderRecorder final : public ThreadSafeRefCounted<BytecodeOrderRecorder> {
+    WTF_MAKE_NONCOPYABLE(BytecodeOrderRecorder);
+    WTF_MAKE_TZONE_ALLOCATED(BytecodeOrderRecorder);
+public:
+    static Ref<BytecodeOrderRecorder> create();
+    ~BytecodeOrderRecorder();
+    // In creation order.
+    static Vector<Ref<BytecodeOrderRecorder>> allInProcess();
+    // Null unless the VM is recording, and while a PauseScope is alive on it.
+    static BytecodeOrderRecorder* ifRecording(VM&);
+    // Null unless the VM is recording.
+    static BytecodeOrderRecorder* ofVM(VM&);
+
+    // While one is alive on the recorder's VM (whose thread it belongs to) nothing is recorded: for code that decodes
+    // everything a payload holds rather than what a program uses.
+    class PauseScope {
+        WTF_MAKE_NONCOPYABLE(PauseScope);
+    public:
+        explicit PauseScope(VM& vm)
+            : m_recorder(ofVM(vm))
+        {
+            if (m_recorder)
+                m_recorder->m_pauseDepth++;
+        }
+        ~PauseScope()
+        {
+            if (m_recorder)
+                m_recorder->m_pauseDepth--;
+        }
+
+    private:
+        RefPtr<BytecodeOrderRecorder> m_recorder;
+    };
+
+    // Code is known by where its record is in its payload: what an order file names it by takes more of the payload than
+    // the program has decoded (BytecodeOrderFile). So only code of payloads that outlive the program
+    // (CachedBytecode::payloadIsPersistent) is recorded; the callers see to that.
+    void didDecodeFunction(const void* record, UnlinkedFunctionCodeBlock&);
+    void didDecodeModule(const void* record);
+    void didReadString(std::span<const uint8_t> stringTable, uint32_t ordinal);
+
+    // While set, on the VM's thread: nothing is recorded, and every function code block that is decoded is told where
+    // its record is.
+    using RecordsOfCodeBlocks = UncheckedKeyHashMap<UnlinkedFunctionCodeBlock*, const void*>;
+    void setRecordsOfCodeBlocks(RecordsOfCodeBlocks* records) { m_recordsOfCodeBlocks = records; }
+
+    struct Snapshot {
+        Vector<const void*> functions;
+        Vector<const void*> modules;
+        std::span<const uint8_t> stringTable; // DecoderStringTable's bytes
+        Vector<uint32_t> stringOrdinals;
+    };
+    Snapshot snapshot() const;
+
+private:
+    BytecodeOrderRecorder();
+
+    mutable Lock m_lock;
+    Snapshot m_recorded WTF_GUARDED_BY_LOCK(m_lock);
+    // A function is decoded again after its code was returned to the cache.
+    UncheckedKeyHashSet<const void*> m_seenFunctions WTF_GUARDED_BY_LOCK(m_lock);
+    UncheckedKeyHashSet<const void*> m_seenModules WTF_GUARDED_BY_LOCK(m_lock);
+    BitVector m_seenStrings WTF_GUARDED_BY_LOCK(m_lock);
+    unsigned m_pauseDepth { 0 }; // the VM's thread only
+    RecordsOfCodeBlocks* m_recordsOfCodeBlocks { nullptr }; // the VM's thread only
+};
+#endif
+
 // The persistent payloads (CachePayload::isPersistent) a VM has code from. An UnlinkedCodeBlock decoded from one
 // remembers its slot here and its record's offset, which is enough to decode it again after it was dropped
 // (UnlinkedFunctionExecutable::returnCodeToCache) without every block holding on to a Decoder.
@@ -148,6 +242,39 @@ class PersistentBytecodePayloads final : public WeakGCHashTable {
     // Before the heap's last finalization takes the weak references' storage.
     void clearChildExecutables() { m_childExecutables.clear(); }
 
+#if USE(BUN_JSC_ADDITIONS)
+    JS_EXPORT_PRIVATE BytecodeOrderRecorder& enableOrderRecording();
+    BytecodeOrderRecorder* orderRecorder() { return m_orderRecorder.get(); }
+
+    // How well the order file a payload was laid out by (BytecodeLinkEncoder) matches what this VM runs: the function
+    // bodies decoded out of the payload, by the region they lie in. A body decoded again after its code was returned to
+    // the cache counts again. `bytes`: a body's own arrays and record, not what it shares with an identical earlier one.
+    struct LinkedPayloadStatistics {
+        struct Bodies {
+            uint64_t count { 0 };
+            uint64_t bytes { 0 };
+        };
+        std::array<uint32_t, BytecodeLinkRegions::Count> regionEnds { };
+        Bodies hot;
+        Bodies unknown;
+        Bodies cold;
+    };
+    // The embedder says which of its payloads is a linked one. Nothing is counted for any other payload.
+    JS_EXPORT_PRIVATE void setLinkedPayload(std::span<const uint8_t>, const std::array<uint32_t, BytecodeLinkRegions::Count>& regionEnds);
+    const LinkedPayloadStatistics* linkedPayloadStatistics() const { return m_linkedPayloadBase ? &m_linkedPayloadStatistics : nullptr; }
+    void didDecodeFunctionBody(const void* payloadBase, uint32_t recordOffset, uint32_t bytes)
+    {
+        if (payloadBase != m_linkedPayloadBase)
+            return;
+        auto& regionEnds = m_linkedPayloadStatistics.regionEnds;
+        auto& bodies = recordOffset < regionEnds[BytecodeLinkRegions::Hot] ? m_linkedPayloadStatistics.hot
+            : recordOffset < regionEnds[BytecodeLinkRegions::Unknown] ? m_linkedPayloadStatistics.unknown
+            : m_linkedPayloadStatistics.cold;
+        bodies.count++;
+        bodies.bytes += bytes;
+    }
+#endif
+
     // For diagnostics: what this and the Decoders it can reach hold on to.
     struct Statistics {
         size_t payloads { 0 };
@@ -176,6 +303,11 @@ class PersistentBytecodePayloads final : public WeakGCHashTable {
     UncheckedKeyHashMap<Identity, uint16_t> m_indices;
     VM& m_vm;
     UncheckedKeyHashMap<uint64_t, FixedVector<Weak<UnlinkedFunctionExecutable>>> m_childExecutables;
+#if USE(BUN_JSC_ADDITIONS)
+    RefPtr<BytecodeOrderRecorder> m_orderRecorder;
+    const void* m_linkedPayloadBase { nullptr }; // never a Decoder's base while null
+    LinkedPayloadStatistics m_linkedPayloadStatistics;
+#endif
 };
 
 } // namespace JSC