[Breaking change] Simplify object store heap size configuration (#697)

**Breaks the previously available startup options --obj-memory and --obj-total-memory**

This change simplifies configuring object store heap size for continuous monitoring. Instead of configuring total object store size, memory constraints for object store can now be configured using the following simple settings.

obj-index - Size of Object store index
obj-log-memory - Size of object store log that contains references to heap objects (Previously available as --obj-memory)
obj-heap-memory - Size of object store heap

Previously available --obj-total-memory option is now removed as object store memory can now be controlled using the above options in a fine grained manner.

Total object store memory footprint = obj-index + obj-log-memory + obj-heap-memory.
No changes to how the main store size is configured.

*  Simplified object store memory configuration
* Added additional details on GC memory to INFO MEMORY command output. This also includes memory that is held by GC not yet released back to the system.
* Bug fix to account for null IGarnetObject.
* Updated documentation.

Author: yrajas

libs/host/Configuration/Options.cs

@@ -52,7 +52,7 @@ internal sealed class Options
         public string SegmentSize { get; set; }
 
         [MemorySizeValidation]
-        [Option('i', "index", Required = false, HelpText = "Size of hash index in bytes (rounds down to power of 2)")]
+        [Option('i', "index", Required = false, HelpText = "Start size of hash index in bytes (rounds down to power of 2)")]
         public string IndexSize { get; set; }
 
         [MemorySizeValidation(false)]
@@ -64,11 +64,11 @@ internal sealed class Options
         public int MutablePercent { get; set; }
 
         [MemorySizeValidation(false)]
-        [Option("obj-total-memory", Required = false, HelpText = "Total object store log memory used including heap memory in bytes")]
-        public string ObjectStoreTotalMemorySize { get; set; }
+        [Option("obj-heap-memory", Required = false, HelpText = "Object store heap memory size in bytes (Sum of size taken up by all object instances in the heap)")]
+        public string ObjectStoreHeapMemorySize { get; set; }
 
         [MemorySizeValidation]
-        [Option("obj-memory", Required = false, HelpText = "Object store log memory used in bytes excluding heap memory")]
+        [Option("obj-log-memory", Required = false, HelpText = "Object store log memory used in bytes (Size of only the log with references to heap objects, excludes size of heap memory consumed by the objects themselves referred to from the log)")]
         public string ObjectStoreLogMemorySize { get; set; }
 
         [MemorySizeValidation]
@@ -80,7 +80,7 @@ internal sealed class Options
         public string ObjectStoreSegmentSize { get; set; }
 
         [MemorySizeValidation]
-        [Option("obj-index", Required = false, HelpText = "Size of object store hash index in bytes (rounds down to power of 2)")]
+        [Option("obj-index", Required = false, HelpText = "Start size of object store hash index in bytes (rounds down to power of 2)")]
         public string ObjectStoreIndexSize { get; set; }
 
         [MemorySizeValidation(false)]
@@ -575,7 +575,7 @@ public GarnetServerOptions GetServerOptions(ILogger logger = null)
                 IndexSize = IndexSize,
                 IndexMaxSize = IndexMaxSize,
                 MutablePercent = MutablePercent,
-                ObjectStoreTotalMemorySize = ObjectStoreTotalMemorySize,
+                ObjectStoreHeapMemorySize = ObjectStoreHeapMemorySize,
                 ObjectStoreLogMemorySize = ObjectStoreLogMemorySize,
                 ObjectStorePageSize = ObjectStorePageSize,
                 ObjectStoreSegmentSize = ObjectStoreSegmentSize,

libs/host/GarnetServer.cs

@@ -198,6 +198,14 @@ private void InitializeServer()
 
             logger?.LogTrace("TLS is {tlsEnabled}", opts.TlsOptions == null ? "disabled" : "enabled");
 
+            if (logger != null)
+            {
+                var configMemoryLimit = (store.IndexSize * 64) + store.Log.MaxMemorySizeBytes + (store.ReadCache?.MaxMemorySizeBytes ?? 0) + (appendOnlyFile?.MaxMemorySizeBytes ?? 0);
+                if (objectStore != null)
+                    configMemoryLimit += objectStore.IndexSize * 64 + objectStore.Log.MaxMemorySizeBytes + (objectStore.ReadCache?.MaxMemorySizeBytes ?? 0) + (objectStoreSizeTracker?.TargetSize ?? 0);
+                logger.LogInformation("Total configured memory limit: {configMemoryLimit}", configMemoryLimit);
+            }
+
             // Create Garnet TCP server if none was provided.
             this.server ??= new GarnetServerTcp(opts.Address, opts.Port, 0, opts.TlsOptions, opts.NetworkSendThrottleMax, logger);
 
@@ -274,7 +282,7 @@ private void CreateObjectStore(IClusterFactory clusterFactory, CustomCommandMana
             if (!opts.DisableObjects)
             {
                 objKvSettings = opts.GetObjectStoreSettings(this.loggerFactory?.CreateLogger("TsavoriteKV  [obj]"),
-                    out var objTotalMemorySize);
+                    out var objHeapMemorySize);
 
                 // Run checkpoint on its own thread to control p99
                 objKvSettings.ThrottleCheckpointFlushDelayMs = opts.CheckpointThrottleFlushDelayMs;
@@ -296,8 +304,8 @@ private void CreateObjectStore(IClusterFactory clusterFactory, CustomCommandMana
                         () => new GarnetObjectSerializer(customCommandManager))
                     , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions));
 
-                if (objTotalMemorySize > 0)
-                    objectStoreSizeTracker = new CacheSizeTracker(objectStore, objKvSettings, objTotalMemorySize,
+                if (objHeapMemorySize > 0)
+                    objectStoreSizeTracker = new CacheSizeTracker(objectStore, objKvSettings, objHeapMemorySize,
                         this.loggerFactory);
             }
         }

libs/host/defaults.conf

@@ -18,7 +18,7 @@
 	/* Size of each log segment in bytes on disk (rounds down to power of 2) */
 	"SegmentSize" : "1g",
 
-	/* Size of hash index in bytes (rounds down to power of 2) */
+	/* Start size of hash index in bytes (rounds down to power of 2) */
 	"IndexSize" : "128m",
 
 	/* Max size of hash index in bytes (rounds down to power of 2) */
@@ -27,10 +27,10 @@
 	/* Percentage of log memory that is kept mutable */
 	"MutablePercent" : 90,
 
-	/* Total object store log memory used including heap memory in bytes */
-	"ObjectStoreTotalMemorySize" : "",
+	/* Object store heap memory size in bytes (Sum of size taken up by all object instances in the heap) */
+	"ObjectStoreHeapMemorySize" : "",
 
-	/* Object store log memory used in bytes excluding heap memory */
+	/* Object store log memory used in bytes (Size of only the log with references to heap objects, excludes size of heap memory consumed by the objects themselves referred to from the log) */
 	"ObjectStoreLogMemorySize" : "32m",
 
 	/* Size of each object store page in bytes (rounds down to power of 2) */
@@ -39,7 +39,7 @@
 	/* Size of each object store log segment in bytes on disk (rounds down to power of 2) */
 	"ObjectStoreSegmentSize" : "32m",
 
-	/* Size of object store hash index in bytes (rounds down to power of 2) */
+	/* Start size of object store hash index in bytes (rounds down to power of 2) */
 	"ObjectStoreIndexSize" : "16m",
 
 	/* Max size of object store hash index in bytes (rounds down to power of 2) */

libs/server/Metrics/Info/GarnetInfoMetrics.cs

@@ -69,19 +69,28 @@ private void PopulateMemoryInfo(StoreWrapper storeWrapper)
             var total_main_store_size = main_store_index_size + main_store_log_memory_size + main_store_read_cache_size;
 
             var object_store_index_size = -1L;
-            var object_store_log_memory_references_size = -1L;
-            var object_store_read_cache_size = -1L;
+            var object_store_log_memory_size = -1L;
+            var object_store_read_cache_log_memory_size = -1L;
+            var object_store_heap_memory_size = -1L;
+            var object_store_read_cache_heap_memory_size = -1L;
             var total_object_store_size = -1L;
             var disableObj = storeWrapper.serverOptions.DisableObjects;
 
+            var aof_log_memory_size = storeWrapper.appendOnlyFile?.MemorySizeBytes ?? -1;
+
             if (!disableObj)
             {
                 object_store_index_size = storeWrapper.objectStore.IndexSize * 64;
-                object_store_log_memory_references_size = storeWrapper.objectStore.Log.MemorySizeBytes;
-                object_store_read_cache_size = (storeWrapper.objectStore.ReadCache != null ? storeWrapper.objectStore.ReadCache.MemorySizeBytes : 0);
-                total_object_store_size = object_store_index_size + object_store_log_memory_references_size + object_store_read_cache_size;
+                object_store_log_memory_size = storeWrapper.objectStore.Log.MemorySizeBytes;
+                object_store_read_cache_log_memory_size = storeWrapper.objectStore.ReadCache?.MemorySizeBytes ?? 0;
+                object_store_heap_memory_size = storeWrapper.objectStoreSizeTracker?.mainLogTracker.LogHeapSizeBytes ?? 0;
+                object_store_read_cache_heap_memory_size = storeWrapper.objectStoreSizeTracker?.readCacheTracker?.LogHeapSizeBytes ?? 0;
+                total_object_store_size = object_store_index_size + object_store_log_memory_size + object_store_read_cache_log_memory_size + object_store_heap_memory_size + object_store_read_cache_heap_memory_size;
             }
 
+            var gcMemoryInfo = GC.GetGCMemoryInfo();
+            var gcAvailableMemory = gcMemoryInfo.TotalCommittedBytes - gcMemoryInfo.HeapSizeBytes;
+
             memoryInfo =
             [
                 new("system_page_size", Environment.SystemPageSize.ToString()),
@@ -105,14 +114,21 @@ private void PopulateMemoryInfo(StoreWrapper storeWrapper)
                 new("proc_physical_memory_size(MB)", SystemMetrics.GetPhysicalMemoryUsage(1 << 20).ToString()),
                 new("proc_peak_physical_memory_size", SystemMetrics.GetPeakPhysicalMemoryUsage().ToString()),
                 new("proc_peak_physical_memory_size(MB)", SystemMetrics.GetPeakPhysicalMemoryUsage(1 << 20).ToString()),
+                new("gc_committed_bytes", gcMemoryInfo.TotalCommittedBytes.ToString()),
+                new("gc_heap_bytes", gcMemoryInfo.HeapSizeBytes.ToString()),
+                new("gc_managed_memory_bytes_excluding_heap", gcAvailableMemory.ToString()),
+                new("gc_fragmented_bytes", gcMemoryInfo.FragmentedBytes.ToString()),
                 new("main_store_index_size", main_store_index_size.ToString()),
                 new("main_store_log_memory_size", main_store_log_memory_size.ToString()),
                 new("main_store_read_cache_size", main_store_read_cache_size.ToString()),
                 new("total_main_store_size", total_main_store_size.ToString()),
                 new("object_store_index_size", object_store_index_size.ToString()),
-                new("object_store_log_memory_references_size", object_store_log_memory_references_size.ToString()),
-                new("object_store_read_cache_size", object_store_read_cache_size.ToString()),
-                new("total_object_store_size", total_object_store_size.ToString())
+                new("object_store_log_memory_size", object_store_log_memory_size.ToString()),
+                new("object_store_read_cache_log_memory_size", object_store_read_cache_log_memory_size.ToString()),
+                new("object_store_heap_memory_size", object_store_heap_memory_size.ToString()),
+                new("object_store_read_cache_heap_memory_size", object_store_read_cache_heap_memory_size.ToString()),
+                new("total_object_store_size", total_object_store_size.ToString()),
+                new("aof_memory_size", aof_log_memory_size.ToString())
             ];
         }
 

libs/server/Servers/GarnetServerOptions.cs

@@ -22,9 +22,9 @@ public class GarnetServerOptions : ServerOptions
         public bool DisableObjects = false;
 
         /// <summary>
-        /// Total memory size limit of object store including heap memory of objects.
+        /// Heap memory size limit of object store.
         /// </summary>
-        public string ObjectStoreTotalMemorySize = "";
+        public string ObjectStoreHeapMemorySize = "";
 
         /// <summary>
         /// Object store log memory used in bytes excluding heap memory.
@@ -528,7 +528,7 @@ public static int MemorySizeBits(string memorySize, string storePageSize, out in
         /// <summary>
         /// Get KVSettings for the object store log
         /// </summary>
-        public KVSettings<byte[], IGarnetObject> GetObjectStoreSettings(ILogger logger, out long objTotalMemorySize)
+        public KVSettings<byte[], IGarnetObject> GetObjectStoreSettings(ILogger logger, out long objHeapMemorySize)
         {
             if (ObjectStoreMutablePercent is < 10 or > 95)
                 throw new Exception("ObjectStoreMutablePercent must be between 10 and 95");
@@ -576,8 +576,8 @@ public KVSettings<byte[], IGarnetObject> GetObjectStoreSettings(ILogger logger,
             }
             logger?.LogInformation("[Object Store] Using log mutable percentage of {ObjectStoreMutablePercent}%", ObjectStoreMutablePercent);
 
-            objTotalMemorySize = ParseSize(ObjectStoreTotalMemorySize);
-            logger?.LogInformation("[Object Store] Total memory size including heap objects is {totalMemorySize}", (objTotalMemorySize > 0 ? PrettySize(objTotalMemorySize) : "unlimited"));
+            objHeapMemorySize = ParseSize(ObjectStoreHeapMemorySize);
+            logger?.LogInformation("[Object Store] Total memory size including heap objects is {totalMemorySize}", (objHeapMemorySize > 0 ? PrettySize(objHeapMemorySize) : "unlimited"));
 
             if (EnableStorageTier)
             {

libs/server/Storage/Functions/ObjectStore/RMWMethods.cs

@@ -171,6 +171,7 @@ public bool PostCopyUpdater(ref byte[] key, ref ObjectInput input, ref IGarnetOb
         {
             // We're performing the object update here (and not in CopyUpdater) so that we are guaranteed that 
             // the record was CASed into the hash chain before it gets modified
+            var oldValueSize = oldValue.Size;
             oldValue.CopyUpdate(ref oldValue, ref value, rmwInfo.RecordInfo.IsInNewVersion);
 
             functionsState.watchVersionMap.IncrementVersion(rmwInfo.KeyHash);
@@ -220,7 +221,9 @@ public bool PostCopyUpdater(ref byte[] key, ref ObjectInput input, ref IGarnetOb
                     }
             }
 
-            functionsState.objectStoreSizeTracker?.AddTrackedSize(MemoryUtils.CalculateKeyValueSize(key, value));
+            // If oldValue has been set to null, subtract it's size from the tracked heap size
+            var sizeAdjustment = oldValue == null ? value.Size - oldValueSize : value.Size;
+            functionsState.objectStoreSizeTracker?.AddTrackedSize(sizeAdjustment);
 
             if (functionsState.appendOnlyFile != null)
                 WriteLogRMW(ref key, ref input, rmwInfo.Version, rmwInfo.SessionID);

libs/server/Storage/SizeTracker/CacheSizeTracker.cs

@@ -22,13 +22,11 @@ public class CacheSizeTracker
     {
         internal readonly LogSizeTracker<byte[], IGarnetObject, ObjectStoreFunctions, ObjectStoreAllocator, LogSizeCalculator> mainLogTracker;
         internal readonly LogSizeTracker<byte[], IGarnetObject, ObjectStoreFunctions, ObjectStoreAllocator, LogSizeCalculator> readCacheTracker;
-        internal long targetSize;
+        public long TargetSize;
 
         private const int deltaFraction = 10; // 10% of target size
         private TsavoriteKV<byte[], IGarnetObject, ObjectStoreFunctions, ObjectStoreAllocator> store;
 
-        internal long IndexSizeBytes => store.IndexSize * 64 + store.OverflowBucketCount * 64;
-
         internal bool Stopped => mainLogTracker.Stopped && (readCacheTracker == null || readCacheTracker.Stopped);
 
         /// <summary>Helps calculate size of a record including heap memory in Object store.</summary>
@@ -44,7 +42,7 @@ public readonly long CalculateRecordSize(RecordInfo recordInfo, byte[] key, IGar
             {
                 long size = Utility.RoundUp(key.Length, IntPtr.Size) + MemoryUtils.ByteArrayOverhead;
 
-                if (!recordInfo.Tombstone) // ignore deleted values being evicted (they are accounted for by ConcurrentDeleter)
+                if (!recordInfo.Tombstone && value != null) // ignore deleted values being evicted (they are accounted for by ConcurrentDeleter)
                     size += value.Size;
 
                 return size;
@@ -64,6 +62,7 @@ public CacheSizeTracker(TsavoriteKV<byte[], IGarnetObject, ObjectStoreFunctions,
             Debug.Assert(targetSize > 0);
 
             this.store = store;
+            this.TargetSize = targetSize;
             var logSizeCalculator = new LogSizeCalculator();
 
             var (mainLogTargetSizeBytes, readCacheTargetSizeBytes) = CalculateLogTargetSizeBytes(targetSize);
@@ -96,13 +95,8 @@ public void Start(CancellationToken token)
         /// <param name="newTargetSize">Target size</param>
         public (long mainLogSizeBytes, long readCacheSizeBytes) CalculateLogTargetSizeBytes(long newTargetSize)
         {
-            long residual = newTargetSize - IndexSizeBytes;
-
-            if (residual <= 0)
-                throw new TsavoriteException($"Target size {newTargetSize} must be larger than index size {IndexSizeBytes}");
-
-            var mainLogSizeBytes = this.store.ReadCache == null ? residual : residual / 2;
-            var readCacheSizeBytes = this.store.ReadCache == null ? 0 : residual / 2;
+            var mainLogSizeBytes = this.store.ReadCache == null ? newTargetSize : newTargetSize / 2;
+            var readCacheSizeBytes = this.store.ReadCache == null ? 0 : newTargetSize / 2;
 
             return (mainLogSizeBytes, readCacheSizeBytes);
         }

libs/storage/Tsavorite/cs/src/core/Allocator/AllocatorBase.cs

@@ -714,6 +714,9 @@ protected void Initialize(long firstValidAddress)
         /// <summary>Minimum number of empty pages in circular buffer to be maintained to account for non-power-of-two size</summary>
         public int MinEmptyPageCount;
 
+        /// <summary>Maximum memory size in bytes</summary>
+        public long MaxMemorySizeBytes => (BufferSize - MinEmptyPageCount) * (long)PageSize;
+
         /// <summary>How many pages do we leave empty in the in-memory buffer (between 0 and BufferSize-1)</summary>
         public int EmptyPageCount
         {

libs/storage/Tsavorite/cs/src/core/Index/Common/LogSizeTracker.cs

@@ -197,11 +197,11 @@ async Task ResizerTask(CancellationToken token)
         /// </summary>
         private void ResizeIfNeeded(CancellationToken token)
         {
-            // Include memory size from the log (logAccessor.MemorySizeBytes) + heap size (logSize.Total) to check utilization
-            if (logSize.Total + logAccessor.MemorySizeBytes > highTargetSize)
+            // Monitor the heap size
+            if (logSize.Total > highTargetSize)
             {
-                logger?.LogDebug("Heap size {totalLogSize} + log {MemorySizeBytes} > target {highTargetSize}. Alloc: {AllocatedPageCount} EPC: {EmptyPageCount}", logSize.Total, logAccessor.MemorySizeBytes, highTargetSize, logAccessor.AllocatedPageCount, logAccessor.EmptyPageCount);
-                while (logSize.Total + logAccessor.MemorySizeBytes > highTargetSize &&
+                logger?.LogDebug("Heap size {totalLogSize} > target {highTargetSize}. Alloc: {AllocatedPageCount} EPC: {EmptyPageCount}", logSize.Total, highTargetSize, logAccessor.AllocatedPageCount, logAccessor.EmptyPageCount);
+                while (logSize.Total > highTargetSize &&
                     logAccessor.EmptyPageCount < logAccessor.MaxEmptyPageCount)
                 {
                     token.ThrowIfCancellationRequested();
@@ -216,10 +216,10 @@ private void ResizeIfNeeded(CancellationToken token)
                     logger?.LogDebug("Increasing empty page count to {EmptyPageCount}", logAccessor.EmptyPageCount);
                 }
             }
-            else if (logSize.Total + logAccessor.MemorySizeBytes < lowTargetSize)
+            else if (logSize.Total < lowTargetSize)
             {
-                logger?.LogDebug("Heap size {totalLogSize} + log {MemorySizeBytes} < target {lowTargetSize}. Alloc: {AllocatedPageCount} EPC: {EmptyPageCount}", logSize.Total, logAccessor.MemorySizeBytes, lowTargetSize, logAccessor.AllocatedPageCount, logAccessor.EmptyPageCount);
-                while (logSize.Total + logAccessor.MemorySizeBytes < lowTargetSize &&
+                logger?.LogDebug("Heap size {totalLogSize} < target {lowTargetSize}. Alloc: {AllocatedPageCount} EPC: {EmptyPageCount}", logSize.Total, lowTargetSize, logAccessor.AllocatedPageCount, logAccessor.EmptyPageCount);
+                while (logSize.Total < lowTargetSize &&
                     logAccessor.EmptyPageCount > logAccessor.MinEmptyPageCount)
                 {
                     token.ThrowIfCancellationRequested();

libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/LogAccessor.cs

@@ -107,6 +107,11 @@ public void SetEmptyPageCount(int pageCount, bool wait = false)
         /// </summary>
         public long MemorySizeBytes => ((long)(allocatorBase.AllocatedPageCount + allocator.OverflowPageCount)) << allocatorBase.LogPageSizeBits;
 
+        /// <summary>
+        /// Maximum memory size in bytes
+        /// </summary>
+        public long MaxMemorySizeBytes => allocatorBase.MaxMemorySizeBytes;
+
         /// <summary>
         /// Number of pages allocated
         /// </summary>