[JSC] CodeBlock aging: let idle FTL code (and counter-less DFG code) age out once the mutator goes quiet

JSTests/stress/codeblock-aging-ftl-idle-off.js

@@ -0,0 +1,20 @@
+//@ runDefault("--useEagerCodeBlockJettisonTiming=1", "--useConcurrentJIT=0", "--optimizedCodeAgingQuietAllocationMB=0")
+
+// With the option off, FTL code never ages out (the behaviour before optimizedCodeAgingQuietAllocationMB).
+
+function shouldBe(actual, expected, msg) {
+    if (actual !== expected)
+        throw new Error((msg || "") + " expected " + expected + " but got " + actual);
+}
+
+var f = new Function("o", "var s = 0; for (var i = 0; i < 8; i++) s += o.a + o.b * i; return [s, isFinalTier()];");
+var o = { a: 1, b: 2 }, reached = false;
+for (var i = 0; i < 1e6 && !reached; i++)
+    reached = f(o)[1];
+if (!reached)
+    throw new Error("test needs f to reach FTL");
+fullGC();
+var start = preciseTime();
+while (preciseTime() - start < 0.2) { }
+fullGC();
+shouldBe(f(o)[1], true, "with the option off an FTL block never ages out;");

JSTests/stress/codeblock-aging-ftl-idle.js

@@ -0,0 +1,69 @@
+//@ runDefault("--useEagerCodeBlockJettisonTiming=1", "--useConcurrentJIT=0")
+
+// FTL code has no execution counter of its own, so it ages against the mutator as a whole
+// (Options::optimizedCodeAgingQuietAllocationMB): once past its TTL it is jettisoned by a full
+// collection that finds (almost) nothing was allocated since the previous one looked at it, and
+// kept - together with the baseline code it pins - whenever the mutator has been allocating.
+
+function shouldBe(actual, expected, msg) {
+    if (actual !== expected)
+        throw new Error((msg || "") + " expected " + expected + " but got " + actual);
+}
+
+function makeHot() {
+    return new Function("o", "var s = 0; for (var i = 0; i < 8; i++) s += o.a + o.b * i; return [s, isFinalTier()];");
+}
+
+function warmToFTL(f) {
+    var o = { a: 1, b: 2 };
+    for (var i = 0; i < 1e6; i++) {
+        if (f(o)[1])
+            return true;
+    }
+    return false;
+}
+
+function idleFor(seconds) {
+    var start = preciseTime();
+    while (preciseTime() - start < seconds) { }
+}
+
+function allocateMB(n) {
+    var keep = [];
+    for (var i = 0; i < n * 16; i++)
+        keep.push(new Float64Array(8192)); // 64 KB each
+    return keep.length;
+}
+
+// The eager FTL TTL is 120ms.
+var ttl = 0.2;
+var o = { a: 1, b: 2 };
+
+// 1. Nothing is allocated between two full collections a TTL apart: the FTL block goes.
+var f = makeHot();
+if (!warmToFTL(f))
+    throw new Error("test needs f to reach FTL");
+fullGC();          // records the allocation total for f's FTL block
+idleFor(ttl);
+fullGC();          // nothing allocated since, past TTL: aged out
+shouldBe(f(o)[1], false, "an FTL block should age out across an idle stretch;");
+shouldBe(f(o)[0], 64, "and the function should still work;");
+
+// 2. The mutator allocates between the collections: the block stays, however old.
+var g = makeHot();
+if (!warmToFTL(g))
+    throw new Error("test needs g to reach FTL");
+fullGC();
+for (var round = 0; round < 5; round++) {
+    idleFor(ttl);
+    allocateMB(4);
+    fullGC();
+    shouldBe(g(o)[1], true, "an FTL block should survive while the mutator is allocating (round " + round + ");");
+}
+
+// 3. And once the mutator goes quiet it goes too.
+idleFor(ttl);
+fullGC();
+idleFor(ttl);
+fullGC();
+shouldBe(g(o)[1], false, "the FTL block should age out once the mutator goes quiet;");

Source/JavaScriptCore/bytecode/CodeBlock.cpp

@@ -863,6 +863,7 @@ void CodeBlock::setupWithUnlinkedBaselineCode(Ref<BaselineJITCode> jitCode)
         setBaselineJITData(WTF::move(baselineJITData));
 #if USE(BUN_JSC_ADDITIONS)
         m_previousCounter = static_cast<BaselineJITData*>(m_jitData)->executeCounter().count();
+        m_previousCounterHoldsAllocation = false;
 #endif
 
         // Set optimization thresholds only after instructions is initialized and JITData is initialized, since these
@@ -1342,6 +1343,22 @@ Seconds CodeBlock::timeToLive(JITType jitType)
     }
 }
 
+#if USE(BUN_JSC_ADDITIONS)
+bool CodeBlock::agesByMutatorQuietness()
+{
+    switch (jitType()) {
+    case JITType::FTLJIT:
+        return true;
+#if ENABLE(DFG_JIT)
+    case JITType::DFGJIT:
+        return !Options::useExecutionCountForCodeBlockAging() || !dfgJITData() || baselineVersion()->m_didFailFTLCompilation;
+#endif
+    default:
+        return false;
+    }
+}
+#endif
+
 template<typename Visitor>
 ALWAYS_INLINE bool CodeBlock::shouldJettisonDueToOldAge(const ConcurrentJSLocker&, Visitor& visitor)
 {
@@ -1354,19 +1371,34 @@ ALWAYS_INLINE bool CodeBlock::shouldJettisonDueToOldAge(const ConcurrentJSLocker
 #if USE(BUN_JSC_ADDITIONS)
     JITType type = jitType();
     Seconds ttl = timeToLive(type);
-    if (timeSinceCreation() < ttl)
-        return false;
 
-    // Optimizing tiers: an FTL block is small and very expensive to rebuild, so it never ages out (it stays alive for as
-    // long as the structures it speculated on do, see determineLiveness()). A DFG block ages like a baseline one, using
-    // the tier-up counter its code already decrements at returns and loop back-edges as the sign of life; a DFG block
-    // compiled without tier-up checks has no such signal and is kept.
-    if (type == JITType::FTLJIT)
-        return false;
-#if ENABLE(DFG_JIT)
-    if (type == JITType::DFGJIT && (!Options::useExecutionCountForCodeBlockAging() || !dfgJITData() || baselineVersion()->m_didFailFTLCompilation))
+    // Optimizing tiers: a DFG block ages like a baseline one, using the tier-up counter its code already decrements at
+    // returns and loop back-edges as the sign of life. FTL code, and DFG code compiled without tier-up checks, has no
+    // counter of its own. It is expensive to rebuild, but it also pins its baseline alternative and every baseline
+    // CodeBlock it inlined along with their metadata, so rather than never ageing it ages against the mutator as a
+    // whole: a full collection lets it go once it is past its TTL and (almost) nothing has been allocated since the
+    // previous full collection looked at it - the process has been sitting idle - and renews it whenever the mutator
+    // has been allocating, exactly as before. Collections are themselves allocation-driven, so a busy mutator never
+    // sees two such quiet collections in a row; an embedder's idle-time collections do.
+    if (agesByMutatorQuietness()) {
+        unsigned quietMB = Options::optimizedCodeAgingQuietAllocationMB();
+        if (!quietMB)
+            return false;
+        // Whole megabytes so the float holds them exactly.
+        float allocatedMB = static_cast<float>(vm().heap.totalBytesAllocated() >> 20);
+        if (!m_previousCounterHoldsAllocation || allocatedMB - m_previousCounter > quietMB) {
+            // First look at this block in this regime, or the mutator has been allocating since its lease started: renew it from here.
+            m_previousCounter = allocatedMB;
+            m_previousCounterHoldsAllocation = true;
+            m_creationTime = ApproximateTime::now();
+            return false;
+        }
+        Seconds quietFor = Options::useEagerCodeBlockJettisonTiming() ? ttl : Seconds(Options::optimizedCodeAgingQuietSeconds());
+        return timeSinceCreation() >= quietFor;
+    }
+
+    if (timeSinceCreation() < ttl)
         return false;
-#endif
 
     if (Options::useExecutionCountForCodeBlockAging()) {
         // LLInt and Baseline CodeBlocks already tick an execution counter on
@@ -1379,7 +1411,9 @@ ALWAYS_INLINE bool CodeBlock::shouldJettisonDueToOldAge(const ConcurrentJSLocker
         // reconcileWeakReferencesAtGCEnd also writes for UnlinkedCodeBlock aging when
         // VM::useUnlinkedCodeBlockJettisoning() is enabled. Both sites store the
         // same current count for the same tier, so they agree; outside that mode
-        // updateActivity() never touches the field.
+        // updateActivity() never touches the field. (For agesByMutatorQuietness() code
+        // the field holds an allocation total instead, tagged by m_previousCounterHoldsAllocation,
+        // and updateActivity() leaves it alone.)
         float currentCount = 0;
         bool hasCounter = false;
         switch (type) {
@@ -1404,8 +1438,9 @@ ALWAYS_INLINE bool CodeBlock::shouldJettisonDueToOldAge(const ConcurrentJSLocker
         default:
             break;
         }
-        if (hasCounter && currentCount != m_previousCounter) {
+        if (hasCounter && (currentCount != m_previousCounter || m_previousCounterHoldsAllocation)) {
             m_previousCounter = currentCount;
+            m_previousCounterHoldsAllocation = false;
             // Push the effective creation time forward so the block is not
             // considered for old-age jettison again until leaseMultiplier * ttl
             // has elapsed with no observed execution.
@@ -1925,6 +1960,13 @@ void CodeBlock::reconcileWeakReferencesAtGCEnd(VM& vm, CollectionScope)
     auto updateActivity = [&] {
         if (!VM::useUnlinkedCodeBlockJettisoning())
             return;
+#if USE(BUN_JSC_ADDITIONS)
+        if (agesByMutatorQuietness()) {
+            if (!m_visitChildrenSkippedDueToOldAge)
+                m_unlinkedCode->resetAge();
+            return;
+        }
+#endif
         auto* jitCode = m_jitCode.get();
         float count = 0;
         bool alwaysActive = false;

Source/JavaScriptCore/bytecode/CodeBlock.h

@@ -825,6 +825,8 @@ class CodeBlock : public JSCell {
     bool m_isJettisoned : 1;
 
     bool m_visitChildrenSkippedDueToOldAge { false };
+    // m_previousCounter holds an execution-counter snapshot, or for agesByMutatorQuietness() code the heap's allocation total (MB) at the lease's start.
+    bool m_previousCounterHoldsAllocation { false };
 
     // Internal methods for use by validation code. It would be private if it wasn't
     // for the fact that we use it from anonymous namespaces.
@@ -930,7 +932,13 @@ class CodeBlock : public JSCell {
 public:
     static Seconds timeToLive(JITType);
     // Start the execution-count aging lease from the counter's current value (call when a tier's code is installed).
-    void snapshotExecutionCounterForAging(float count) { m_previousCounter = count; }
+    void snapshotExecutionCounterForAging(float count)
+    {
+        m_previousCounter = count;
+        m_previousCounterHoldsAllocation = false;
+    }
+    // Optimizing code with no execution counter to read liveness off (FTL; DFG without tier-up checks): it ages once the mutator goes quiet instead.
+    bool agesByMutatorQuietness();
 private:
 
     template<typename Visitor> void propagateTransitions(const ConcurrentJSLocker&, Visitor&);

Source/JavaScriptCore/heap/Heap.cpp

@@ -2754,6 +2754,9 @@ void Heap::setGarbageCollectionTimerEnabled(bool enable)
 constexpr size_t oversizedAllocationThreshold = 64 * KB;
 void Heap::didAllocate(size_t bytes)
 {
+#if USE(BUN_JSC_ADDITIONS)
+    m_totalBytesAllocated += bytes;
+#endif
     if (bytes >= oversizedAllocationThreshold) {
         m_oversizedBytesAllocatedThisCycle += bytes;
         m_lastOversidedAllocationThisCycle = bytes;

Source/JavaScriptCore/heap/Heap.h

@@ -725,6 +725,10 @@ class Heap {
     JS_EXPORT_PRIVATE void deprecatedReportExtraMemorySlowCase(size_t);
 
     size_t totalBytesAllocatedThisCycle() { return m_nonOversizedBytesAllocatedThisCycle + m_oversizedBytesAllocatedThisCycle; }
+#if USE(BUN_JSC_ADDITIONS)
+    // Monotonic; lets a full collection tell whether the mutator did anything since an earlier one (CodeBlock::shouldJettisonDueToOldAge).
+    uint64_t totalBytesAllocated() const { return m_totalBytesAllocated; }
+#endif
 
     bool shouldCollectInCollectorThread(const AbstractLocker&);
     void collectInCollectorThread();
@@ -877,6 +881,9 @@ class Heap {
     const size_t m_ramSize;
     const size_t m_minBytesPerCycle;
     size_t m_bytesAllocatedBeforeLastEdenCollect { 0 };
+#if USE(BUN_JSC_ADDITIONS)
+    uint64_t m_totalBytesAllocated { 0 };
+#endif
     size_t m_sizeAfterLastCollect { 0 };
     size_t m_sizeAfterLastFullCollect { 0 };
     size_t m_sizeBeforeLastFullCollect { 0 };

Source/JavaScriptCore/runtime/OptionsList.h

@@ -86,6 +86,8 @@ bool hasCapacityToUseLargeGigacage();
     v(Bool, verboseFFI, false, Normal, "dataLog on FFI thunk/stub/signature creation"_s)
 #define FOR_EACH_JSC_CODEBLOCK_AGING_OPTION(v) \
     v(Bool, useExecutionCountForCodeBlockAging, true, Normal, "If true, an LLInt/Baseline CodeBlock whose execution counter has advanced since the last old-age check is treated as still in use and its TTL is renewed instead of being jettisoned."_s) \
+    v(Unsigned, optimizedCodeAgingQuietAllocationMB, 1, Normal, "FTL code (and DFG code without a tier-up counter) past its TTL is jettisoned for old age by a full collection that finds at most this much was allocated since the previous one looked at it, i.e. the mutator has been idle. 0 = such code never ages out."_s) \
+    v(Double, optimizedCodeAgingQuietSeconds, 30, Normal, "How long the mutator must have been that quiet, measured from the last collection that saw it allocating, before such code is let go (the tier's eager TTL under useEagerCodeBlockJettisonTiming)."_s) \
     v(Double, codeBlockAgingLeaseMultiplier, 3.0, Normal, "When useExecutionCountForCodeBlockAging proves a CodeBlock is still active, renew its old-age TTL to this many multiples of timeToLive for its tier."_s)
 #define FOR_EACH_JSC_BYTECODE_CACHE_DECODER_OPTION(v) \
     v(Bool, useLeanBytecodeCacheDecoder, true, Normal, "If true, the bytecode cache Decoder skips bookkeeping that is only needed for decoded objects shared by multiple references."_s) \