CodeCache: function-executable cache entry + depth-bounded eager generation

Source/JavaScriptCore/bytecode/HandlerInfo.h

@@ -85,12 +85,18 @@ struct HandlerInfoBase {
     uint32_t start;
     uint32_t end;
     uint32_t target;
-    uint32_t typeBits : 2; // HandlerType
+    // A full-width field rather than `: 2`: the unused bits of a bitfield are
+    // padding the constructors never write, and UnlinkedHandlerInfo is serialized
+    // bytewise into the bytecode cache, which must be reproducible.
+    uint32_t typeBits { 0 }; // HandlerType
 };
 
 struct UnlinkedHandlerInfo : public HandlerInfoBase {
     UnlinkedHandlerInfo()
     {
+        start = 0;
+        end = 0;
+        target = 0;
     }
 
     UnlinkedHandlerInfo(uint32_t start, uint32_t end, uint32_t target, HandlerType handlerType)

Source/JavaScriptCore/runtime/CachedTypes.cpp

@@ -208,7 +208,7 @@ class Encoder {
     class Page {
     public:
         Page(size_t size)
-            : m_buffer(MallocSpan<uint8_t, VMMalloc>::malloc(size))
+            : m_buffer(MallocSpan<uint8_t, VMMalloc>::zeroedMalloc(size)) // zeroed: alignment padding must be deterministic (blob is hashed/embedded)
         {
         }
 
@@ -2254,6 +2254,7 @@ enum class CachedCodeBlockTag {
     CachedProgramCodeBlockTag,
     CachedModuleCodeBlockTag,
     CachedEvalCodeBlockTag,
+    CachedFunctionExecutableTag,
 };
 
 static CachedCodeBlockTag NODELETE tagFromSourceCodeType(SourceCodeType type)
@@ -2266,7 +2267,7 @@ static CachedCodeBlockTag NODELETE tagFromSourceCodeType(SourceCodeType type)
     case SourceCodeType::ModuleType:
         return CachedCodeBlockTag::CachedModuleCodeBlockTag;
     case SourceCodeType::FunctionType:
-        break;
+        return CachedCodeBlockTag::CachedFunctionExecutableTag;
     }
     ASSERT_NOT_REACHED();
     return static_cast<CachedCodeBlockTag>(-1);
@@ -2573,6 +2574,7 @@ class CachedSourceCodeKey : public CachedObject<SourceCodeKey> {
 class GenericCacheEntry {
 public:
     bool decode(Decoder&, std::pair<SourceCodeKey, UnlinkedCodeBlock*>&) const;
+    bool decode(Decoder&, std::pair<SourceCodeKey, UnlinkedFunctionExecutable*>&) const;
     bool decode(Decoder&, SourceCodeKey&) const;
     bool isStillValid(Decoder&, const SourceCodeKey&, CachedCodeBlockTag) const;
 
@@ -2649,6 +2651,54 @@ class CacheEntry : public GenericCacheEntry {
 static_assert(alignof(CacheEntry<UnlinkedProgramCodeBlock>) <= alignof(std::max_align_t));
 static_assert(alignof(CacheEntry<UnlinkedModuleProgramCodeBlock>) <= alignof(std::max_align_t));
 
+// A top-level cache entry whose payload is a single UnlinkedFunctionExecutable and the
+// tree of UnlinkedFunctionCodeBlocks reachable from it. Programs, modules and eval are
+// cached as code blocks because that is their top-level compilation unit; a builtin's
+// top-level unit is a function, which has no code-block-shaped entry of its own.
+class FunctionExecutableCacheEntry : public GenericCacheEntry {
+public:
+    FunctionExecutableCacheEntry(Encoder& encoder)
+        : GenericCacheEntry(encoder, CachedCodeBlockTag::CachedFunctionExecutableTag)
+    {
+    }
+
+    void encode(Encoder& encoder, std::pair<SourceCodeKey, const UnlinkedFunctionExecutable*> pair)
+    {
+        m_key.encode(encoder, pair.first);
+        m_executable.encode(encoder, pair.second);
+    }
+
+private:
+    friend GenericCacheEntry;
+
+    bool isStillValid(Decoder& decoder, const SourceCodeKey& key) const
+    {
+        SourceCodeKey decodedKey;
+        m_key.decode(decoder, decodedKey);
+        return decodedKey == key;
+    }
+
+    bool decode(Decoder& decoder, std::pair<SourceCodeKey, UnlinkedFunctionExecutable*>& result) const
+    {
+        ASSERT(tag() == CachedCodeBlockTag::CachedFunctionExecutableTag);
+        SourceCodeKey decodedKey;
+        m_key.decode(decoder, decodedKey);
+        result = { WTF::move(decodedKey), m_executable.decode(decoder) };
+        return true;
+    }
+
+    bool decode(Decoder& decoder, SourceCodeKey& key) const
+    {
+        m_key.decode(decoder, key);
+        return true;
+    }
+
+    CachedSourceCodeKey m_key;
+    CachedPtr<CachedFunctionExecutable> m_executable;
+};
+
+static_assert(alignof(FunctionExecutableCacheEntry) <= alignof(std::max_align_t));
+
 bool GenericCacheEntry::decode(Decoder& decoder, std::pair<SourceCodeKey, UnlinkedCodeBlock*>& result) const
 {
     if (!isUpToDate(decoder))
@@ -2659,6 +2709,9 @@ bool GenericCacheEntry::decode(Decoder& decoder, std::pair<SourceCodeKey, Unlink
         return std::bit_cast<const CacheEntry<UnlinkedProgramCodeBlock>*>(this)->decode(decoder, reinterpret_cast<std::pair<SourceCodeKey, UnlinkedProgramCodeBlock*>&>(result));
     case CachedCodeBlockTag::CachedModuleCodeBlockTag:
         return std::bit_cast<const CacheEntry<UnlinkedModuleProgramCodeBlock>*>(this)->decode(decoder, reinterpret_cast<std::pair<SourceCodeKey, UnlinkedModuleProgramCodeBlock*>&>(result));
+    case CachedCodeBlockTag::CachedFunctionExecutableTag:
+        // The caller asked for a code block; this entry holds a function executable.
+        return false;
     case CachedCodeBlockTag::CachedEvalCodeBlockTag:
         // We do not cache eval code blocks
         RELEASE_ASSERT_NOT_REACHED();
@@ -2667,6 +2720,17 @@ bool GenericCacheEntry::decode(Decoder& decoder, std::pair<SourceCodeKey, Unlink
     return false;
 }
 
+bool GenericCacheEntry::decode(Decoder& decoder, std::pair<SourceCodeKey, UnlinkedFunctionExecutable*>& result) const
+{
+    if (!isUpToDate(decoder))
+        return false;
+
+    if (m_tag != CachedCodeBlockTag::CachedFunctionExecutableTag)
+        return false;
+
+    return std::bit_cast<const FunctionExecutableCacheEntry*>(this)->decode(decoder, result);
+}
+
 bool GenericCacheEntry::decode(Decoder& decoder, SourceCodeKey& key) const
 {
     if (!isUpToDate(decoder))
@@ -2677,6 +2741,8 @@ bool GenericCacheEntry::decode(Decoder& decoder, SourceCodeKey& key) const
         return std::bit_cast<const CacheEntry<UnlinkedProgramCodeBlock>*>(this)->decode(decoder, key);
     case CachedCodeBlockTag::CachedModuleCodeBlockTag:
         return std::bit_cast<const CacheEntry<UnlinkedModuleProgramCodeBlock>*>(this)->decode(decoder, key);
+    case CachedCodeBlockTag::CachedFunctionExecutableTag:
+        return std::bit_cast<const FunctionExecutableCacheEntry*>(this)->decode(decoder, key);
     case CachedCodeBlockTag::CachedEvalCodeBlockTag:
         // We do not cache eval code blocks
         return false;
@@ -2695,6 +2761,8 @@ bool GenericCacheEntry::isStillValid(Decoder& decoder, const SourceCodeKey& key,
         return std::bit_cast<const CacheEntry<UnlinkedProgramCodeBlock>*>(this)->isStillValid(decoder, key);
     case CachedCodeBlockTag::CachedModuleCodeBlockTag:
         return std::bit_cast<const CacheEntry<UnlinkedModuleProgramCodeBlock>*>(this)->isStillValid(decoder, key);
+    case CachedCodeBlockTag::CachedFunctionExecutableTag:
+        return std::bit_cast<const FunctionExecutableCacheEntry*>(this)->isStillValid(decoder, key);
     case CachedCodeBlockTag::CachedEvalCodeBlockTag:
         // We do not cache eval code blocks
         RELEASE_ASSERT_NOT_REACHED();
@@ -2740,6 +2808,33 @@ RefPtr<CachedBytecode> encodeFunctionCodeBlock(VM& vm, const UnlinkedFunctionCod
     return encoder.release(error);
 }
 
+RefPtr<CachedBytecode> encodeFunctionExecutable(VM& vm, const SourceCodeKey& key, const UnlinkedFunctionExecutable* executable, BytecodeCacheError& error)
+{
+    FileSystem::FileHandle invalidFileHandle;
+    Encoder encoder(vm, invalidFileHandle);
+    encoder.malloc<FunctionExecutableCacheEntry>(encoder)->encode(encoder, { key, executable });
+    return encoder.release(error);
+}
+
+UnlinkedFunctionExecutable* decodeFunctionExecutable(VM& vm, const SourceCodeKey& key, Ref<CachedBytecode> cachedBytecode)
+{
+    if (cachedBytecode->size() < sizeof(FunctionExecutableCacheEntry))
+        return nullptr;
+
+    auto* cachedEntry = std::bit_cast<const GenericCacheEntry*>(cachedBytecode->span().data());
+    Ref decoder = Decoder::create(vm, WTF::move(cachedBytecode), &key.source().provider());
+    std::pair<SourceCodeKey, UnlinkedFunctionExecutable*> entry;
+    {
+        DeferGC deferGC(vm);
+        if (!cachedEntry->decode(decoder.get(), entry))
+            return nullptr;
+    }
+    if (entry.first != key)
+        return nullptr;
+
+    return entry.second;
+}
+
 std::optional<SourceCodeKey> decodeSourceCodeKey(VM& vm, Ref<CachedBytecode> cachedBytecode)
 {
     const auto* cachedEntry = std::bit_cast<const GenericCacheEntry*>(cachedBytecode->span().data());

Source/JavaScriptCore/runtime/CachedTypes.h

@@ -128,6 +128,19 @@ JS_EXPORT_PRIVATE RefPtr<CachedBytecode> encodeFunctionCodeBlock(VM&, const Unli
 
 JS_EXPORT_PRIVATE void decodeFunctionCodeBlock(Decoder&, int32_t cachedFunctionCodeBlockOffset, WriteBarrier<UnlinkedFunctionCodeBlock>&, const JSCell*);
 
+// Serialize an UnlinkedFunctionExecutable together with the tree of
+// UnlinkedFunctionCodeBlocks already generated beneath it, as a self-validating
+// top-level cache entry. `key` must use SourceCodeType::FunctionType, and the
+// SourceCode it carries is the executable's parent source.
+//
+// This exists for the JS builtins, whose top-level compilation unit is a builtin
+// function rather than a program or a module. decodeFunctionExecutable() returns
+// nullptr when the entry is stale or does not match `key`; the caller is then
+// expected to compile from source as usual.
+JS_EXPORT_PRIVATE RefPtr<CachedBytecode> encodeFunctionExecutable(VM&, const SourceCodeKey&, const UnlinkedFunctionExecutable*, BytecodeCacheError&);
+
+JS_EXPORT_PRIVATE UnlinkedFunctionExecutable* decodeFunctionExecutable(VM&, const SourceCodeKey&, Ref<CachedBytecode>);
+
 bool isCachedBytecodeStillValid(VM&, Ref<CachedBytecode>, const SourceCodeKey&, SourceCodeType);
 
 } // namespace JSC

Source/JavaScriptCore/runtime/CodeCache.cpp

@@ -57,16 +57,23 @@ void CodeCacheMap::pruneSlowCase()
     }
 }
 
-static void generateUnlinkedCodeBlockForFunctions(VM& vm, UnlinkedCodeBlock* unlinkedCodeBlock, const SourceCode& parentSource, OptionSet<CodeGenerationMode> codeGenerationMode, ParserError& error)
+// `depth` bounds how many levels of nested functions get eagerly compiled: 0 stops at
+// the code block passed in (no nested functions), 1 additionally compiles its immediate
+// children, and so on. maxBytecodeCacheDepth (unbounded) preserves the historical
+// recursive behavior.
+static void generateUnlinkedCodeBlockForFunctions(VM& vm, UnlinkedCodeBlock* unlinkedCodeBlock, const SourceCode& parentSource, OptionSet<CodeGenerationMode> codeGenerationMode, ParserError& error, unsigned depth)
 {
+    if (!depth)
+        return;
+    unsigned childDepth = depth == maxBytecodeCacheDepth ? depth : depth - 1;
     auto generate = [&](UnlinkedFunctionExecutable* unlinkedExecutable, CodeSpecializationKind constructorKind) {
         if (constructorKind == CodeSpecializationKind::CodeForConstruct && SourceParseModeSet(SourceParseMode::AsyncArrowFunctionMode, SourceParseMode::AsyncMethodMode, SourceParseMode::AsyncFunctionMode).contains(unlinkedExecutable->parseMode()))
             return;
 
         SourceCode source = unlinkedExecutable->linkedSourceCode(parentSource);
         UnlinkedFunctionCodeBlock* unlinkedFunctionCodeBlock = unlinkedExecutable->unlinkedCodeBlockFor(vm, source, constructorKind, codeGenerationMode, error, unlinkedExecutable->parseMode());
         if (unlinkedFunctionCodeBlock)
-            generateUnlinkedCodeBlockForFunctions(vm, unlinkedFunctionCodeBlock, source, codeGenerationMode, error);
+            generateUnlinkedCodeBlockForFunctions(vm, unlinkedFunctionCodeBlock, source, codeGenerationMode, error, childDepth);
     };
 
     // FIXME: We should also generate CodeBlocks for CodeForConstruct
@@ -149,27 +156,33 @@ UnlinkedEvalCodeBlock* generateUnlinkedCodeBlockForDirectEval(VM& vm, DirectEval
     return generateUnlinkedCodeBlock<UnlinkedEvalCodeBlock>(vm, executable, source, scriptMode, codeGenerationMode, error, evalContextType, variablesUnderTDZ, privateNameEnvironment);
 }
 
+unsigned bytecodeCacheDepthFromOptions()
+{
+    int32_t depth = Options::bytecodeCacheMaxDepth();
+    return depth < 0 ? maxBytecodeCacheDepth : static_cast<unsigned>(depth);
+}
+
 template <class UnlinkedCodeBlockType>
     requires (!std::same_as<UnlinkedCodeBlockType, UnlinkedEvalCodeBlock>)
-UnlinkedCodeBlockType* recursivelyGenerateUnlinkedCodeBlock(VM& vm, const SourceCode& source, LexicallyScopedFeatures lexicallyScopedFeatures, JSParserScriptMode scriptMode, OptionSet<CodeGenerationMode> codeGenerationMode, ParserError& error, EvalContextType evalContextType)
+UnlinkedCodeBlockType* recursivelyGenerateUnlinkedCodeBlock(VM& vm, const SourceCode& source, LexicallyScopedFeatures lexicallyScopedFeatures, JSParserScriptMode scriptMode, OptionSet<CodeGenerationMode> codeGenerationMode, ParserError& error, EvalContextType evalContextType, unsigned depth)
 {
     bool isArrowFunctionContext = false;
     UnlinkedCodeBlockType* unlinkedCodeBlock = generateUnlinkedCodeBlockImpl<UnlinkedCodeBlockType>(vm, source, lexicallyScopedFeatures, scriptMode, codeGenerationMode, error, evalContextType, DerivedContextType::None, isArrowFunctionContext);
     if (!unlinkedCodeBlock)
         return nullptr;
 
-    generateUnlinkedCodeBlockForFunctions(vm, unlinkedCodeBlock, source, codeGenerationMode, error);
+    generateUnlinkedCodeBlockForFunctions(vm, unlinkedCodeBlock, source, codeGenerationMode, error, depth);
     return unlinkedCodeBlock;
 }
 
-UnlinkedProgramCodeBlock* recursivelyGenerateUnlinkedCodeBlockForProgram(VM& vm, const SourceCode& source, LexicallyScopedFeatures lexicallyScopedFeatures, JSParserScriptMode scriptMode, OptionSet<CodeGenerationMode> codeGenerationMode, ParserError& error, EvalContextType evalContextType)
+UnlinkedProgramCodeBlock* recursivelyGenerateUnlinkedCodeBlockForProgram(VM& vm, const SourceCode& source, LexicallyScopedFeatures lexicallyScopedFeatures, JSParserScriptMode scriptMode, OptionSet<CodeGenerationMode> codeGenerationMode, ParserError& error, EvalContextType evalContextType, unsigned depth)
 {
-    return recursivelyGenerateUnlinkedCodeBlock<UnlinkedProgramCodeBlock>(vm, source, lexicallyScopedFeatures, scriptMode, codeGenerationMode, error, evalContextType);
+    return recursivelyGenerateUnlinkedCodeBlock<UnlinkedProgramCodeBlock>(vm, source, lexicallyScopedFeatures, scriptMode, codeGenerationMode, error, evalContextType, depth);
 }
 
-UnlinkedModuleProgramCodeBlock* recursivelyGenerateUnlinkedCodeBlockForModuleProgram(VM& vm, const SourceCode& source, LexicallyScopedFeatures lexicallyScopedFeatures, JSParserScriptMode scriptMode, OptionSet<CodeGenerationMode> codeGenerationMode, ParserError& error, EvalContextType evalContextType)
+UnlinkedModuleProgramCodeBlock* recursivelyGenerateUnlinkedCodeBlockForModuleProgram(VM& vm, const SourceCode& source, LexicallyScopedFeatures lexicallyScopedFeatures, JSParserScriptMode scriptMode, OptionSet<CodeGenerationMode> codeGenerationMode, ParserError& error, EvalContextType evalContextType, unsigned depth)
 {
-    return recursivelyGenerateUnlinkedCodeBlock<UnlinkedModuleProgramCodeBlock>(vm, source, lexicallyScopedFeatures, scriptMode, codeGenerationMode, error, evalContextType);
+    return recursivelyGenerateUnlinkedCodeBlock<UnlinkedModuleProgramCodeBlock>(vm, source, lexicallyScopedFeatures, scriptMode, codeGenerationMode, error, evalContextType, depth);
 }
 
 template<class UnlinkedCodeBlockType, class ExecutableType>
@@ -325,6 +338,29 @@ SourceCodeKey sourceCodeKeyForSerializedModule(VM& vm, const SourceCode& sourceC
     return sourceCodeKeyForSerializedBytecode(vm, sourceCode, SourceCodeType::ModuleType, StrictModeLexicallyScopedFeature, scriptMode, {});
 }
 
+SourceCodeKey sourceCodeKeyForSerializedFunctionExecutable(VM&, const SourceCode& sourceCode, const String& name)
+{
+    return SourceCodeKey(
+        sourceCode, name, SourceCodeType::FunctionType, NoLexicallyScopedFeatures, JSParserScriptMode::Classic,
+        DerivedContextType::None, EvalContextType::None, false, { },
+        std::nullopt);
+}
+
+UnlinkedFunctionCodeBlock* recursivelyGenerateUnlinkedCodeBlockForFunctionExecutable(VM& vm, UnlinkedFunctionExecutable* executable, const SourceCode& parentSource, ParserError& error, unsigned depth)
+{
+    OptionSet<CodeGenerationMode> codeGenerationMode = { };
+    SourceCode source = executable->linkedSourceCode(parentSource);
+    UnlinkedFunctionCodeBlock* unlinkedCodeBlock = executable->unlinkedCodeBlockFor(vm, source, CodeSpecializationKind::CodeForCall, codeGenerationMode, error, executable->parseMode());
+    if (!unlinkedCodeBlock || error.isValid())
+        return nullptr;
+
+    generateUnlinkedCodeBlockForFunctions(vm, unlinkedCodeBlock, source, codeGenerationMode, error, depth);
+    if (error.isValid())
+        return nullptr;
+
+    return unlinkedCodeBlock;
+}
+
 RefPtr<CachedBytecode> serializeBytecode(VM& vm, UnlinkedCodeBlock* codeBlock, const SourceCode& source, SourceCodeType codeType, LexicallyScopedFeatures lexicallyScopedFeatures, JSParserScriptMode scriptMode, FileSystem::FileHandle& fileHandle, BytecodeCacheError& error, OptionSet<CodeGenerationMode> codeGenerationMode)
 {
     return encodeCodeBlock(vm, sourceCodeKeyForSerializedBytecode(vm, source, codeType, lexicallyScopedFeatures, scriptMode, codeGenerationMode), codeBlock, fileHandle, error);

Source/JavaScriptCore/runtime/CodeCache.h

@@ -26,6 +26,7 @@
 #pragma once
 
 #include "CachedTypes.h"
+#include <limits>
 #include "ExecutableInfo.h"
 #include "JSCInlines.h"
 #include "Parser.h"
@@ -44,6 +45,11 @@
 
 namespace JSC {
 
+// Depth for eager nested-function bytecode generation: 0 = only the top-level code block,
+// N = also its descendants up to N levels; maxBytecodeCacheDepth = unbounded (historical).
+static constexpr unsigned maxBytecodeCacheDepth = std::numeric_limits<unsigned>::max();
+JS_EXPORT_PRIVATE unsigned bytecodeCacheDepthFromOptions();
+
 class EvalExecutable;
 class IndirectEvalExecutable;
 class Identifier;
@@ -263,12 +269,18 @@ template <> struct CacheTypes<UnlinkedModuleProgramCodeBlock> {
 };
 
 UnlinkedEvalCodeBlock* generateUnlinkedCodeBlockForDirectEval(VM&, DirectEvalExecutable*, const SourceCode&, JSParserScriptMode, OptionSet<CodeGenerationMode>, ParserError&, EvalContextType, const TDZEnvironment* variablesUnderTDZ, const PrivateNameEnvironment*);
-UnlinkedProgramCodeBlock* recursivelyGenerateUnlinkedCodeBlockForProgram(VM&, const SourceCode&, LexicallyScopedFeatures, JSParserScriptMode, OptionSet<CodeGenerationMode>, ParserError&, EvalContextType);
-UnlinkedModuleProgramCodeBlock* recursivelyGenerateUnlinkedCodeBlockForModuleProgram(VM&, const SourceCode&, LexicallyScopedFeatures, JSParserScriptMode, OptionSet<CodeGenerationMode>, ParserError&, EvalContextType);
+UnlinkedProgramCodeBlock* recursivelyGenerateUnlinkedCodeBlockForProgram(VM&, const SourceCode&, LexicallyScopedFeatures, JSParserScriptMode, OptionSet<CodeGenerationMode>, ParserError&, EvalContextType, unsigned depth = maxBytecodeCacheDepth);
+UnlinkedModuleProgramCodeBlock* recursivelyGenerateUnlinkedCodeBlockForModuleProgram(VM&, const SourceCode&, LexicallyScopedFeatures, JSParserScriptMode, OptionSet<CodeGenerationMode>, ParserError&, EvalContextType, unsigned depth = maxBytecodeCacheDepth);
+
+// Eagerly compile `executable` and every function nested beneath it, so that the whole
+// tree can be serialized with encodeFunctionExecutable(). `parentSource` is the source
+// the executable was created from, not its own linked sub-range.
+JS_EXPORT_PRIVATE UnlinkedFunctionCodeBlock* recursivelyGenerateUnlinkedCodeBlockForFunctionExecutable(VM&, UnlinkedFunctionExecutable*, const SourceCode& parentSource, ParserError&, unsigned depth = maxBytecodeCacheDepth);
 
 void writeCodeBlock(const SourceCodeKey&, const SourceCodeValue&);
 RefPtr<CachedBytecode> serializeBytecode(VM&, UnlinkedCodeBlock*, const SourceCode&, SourceCodeType, LexicallyScopedFeatures, JSParserScriptMode, FileSystem::FileHandle&, BytecodeCacheError&, OptionSet<CodeGenerationMode>);
 SourceCodeKey sourceCodeKeyForSerializedProgram(VM&, const SourceCode&);
 SourceCodeKey sourceCodeKeyForSerializedModule(VM&, const SourceCode&);
+JS_EXPORT_PRIVATE SourceCodeKey sourceCodeKeyForSerializedFunctionExecutable(VM&, const SourceCode&, const String& name);
 
 } // namespace JSC