diff --git a/src/Nethermind/Nethermind.Consensus/Processing/BlockchainProcessor.cs b/src/Nethermind/Nethermind.Consensus/Processing/BlockchainProcessor.cs index 0469e62fc9f9..816517ff416b 100644 --- a/src/Nethermind/Nethermind.Consensus/Processing/BlockchainProcessor.cs +++ b/src/Nethermind/Nethermind.Consensus/Processing/BlockchainProcessor.cs @@ -143,6 +143,8 @@ public async ValueTask Enqueue(Block block, ProcessingOptions processingOptions) if (!_recoveryComplete) { Interlocked.Increment(ref _queueCount); + BlockAdded?.Invoke(this, new BlockEventArgs(block)); + _lastProcessedBlock = DateTime.UtcNow; try { @@ -424,6 +426,7 @@ private void FireProcessingQueueEmpty() public event EventHandler? ProcessingQueueEmpty; public event EventHandler? BlockRemoved; + public event EventHandler? BlockAdded; public bool IsEmpty => Volatile.Read(ref _queueCount) == 0; public int Count => Volatile.Read(ref _queueCount); diff --git a/src/Nethermind/Nethermind.Consensus/Processing/IBlockProcessingQueue.cs b/src/Nethermind/Nethermind.Consensus/Processing/IBlockProcessingQueue.cs index 7ca72e70d41f..3312d286e23e 100644 --- a/src/Nethermind/Nethermind.Consensus/Processing/IBlockProcessingQueue.cs +++ b/src/Nethermind/Nethermind.Consensus/Processing/IBlockProcessingQueue.cs @@ -26,6 +26,7 @@ public interface IBlockProcessingQueue /// event EventHandler ProcessingQueueEmpty; + event EventHandler BlockAdded; event EventHandler BlockRemoved; /// diff --git a/src/Nethermind/Nethermind.Core.Test/Db/TestMemDbProvider.cs b/src/Nethermind/Nethermind.Core.Test/Db/TestMemDbProvider.cs index b13c69f5aff7..c0fa78e4d9b6 100644 --- a/src/Nethermind/Nethermind.Core.Test/Db/TestMemDbProvider.cs +++ b/src/Nethermind/Nethermind.Core.Test/Db/TestMemDbProvider.cs @@ -10,6 +10,7 @@ using Nethermind.Blockchain.Synchronization; using Nethermind.Db; using Nethermind.Init.Modules; +using Nethermind.Monitoring; namespace Nethermind.Core.Test.Db { @@ -23,7 +24,11 @@ public static Task InitAsync() public static IDbProvider Init() { return new ContainerBuilder() - .AddModule(new DbModule(new InitConfig() { DiagnosticMode = DiagnosticMode.MemDb }, new ReceiptConfig(), new SyncConfig())) + .AddModule(new DbModule( + new InitConfig() { DiagnosticMode = DiagnosticMode.MemDb }, + new ReceiptConfig(), + new SyncConfig() + )) .AddSingleton() .Build() .Resolve(); diff --git a/src/Nethermind/Nethermind.Core/ContainerBuilderExtensions.cs b/src/Nethermind/Nethermind.Core/ContainerBuilderExtensions.cs index 3cad05bac4ba..224300328fa8 100644 --- a/src/Nethermind/Nethermind.Core/ContainerBuilderExtensions.cs +++ b/src/Nethermind/Nethermind.Core/ContainerBuilderExtensions.cs @@ -374,6 +374,15 @@ public static ContainerBuilder Intercept(this ContainerBuilder builder, Actio }); } + public static ContainerBuilder Intercept(this ContainerBuilder builder, Action interceptor) where T : class + { + return builder.AddDecorator((ctx, service) => + { + interceptor(service, ctx); + return service; + }); + } + /// /// A convenient way of creating a service whose members can be configured independent of other instances of the same /// type (assuming the type is of lifetime scope). This is useful for same type with multiple configuration diff --git a/src/Nethermind/Nethermind.Db.Test/DbTrackerTests.cs b/src/Nethermind/Nethermind.Db.Test/DbTrackerTests.cs index 27a1344fc3fb..67d041f9c6b0 100644 --- a/src/Nethermind/Nethermind.Db.Test/DbTrackerTests.cs +++ b/src/Nethermind/Nethermind.Db.Test/DbTrackerTests.cs @@ -1,9 +1,23 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using System; using System.Linq; using Autofac; using FluentAssertions; +using Nethermind.Api; +using Nethermind.Blockchain.Receipts; +using Nethermind.Blockchain.Synchronization; +using Nethermind.Consensus.Processing; using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.Core.Test; +using Nethermind.Core.Test.Builders; +using Nethermind.Init.Modules; +using Nethermind.Logging; +using Nethermind.Monitoring; +using Nethermind.Monitoring.Config; +using NSubstitute; using NUnit.Framework; namespace Nethermind.Db.Test; @@ -14,14 +28,17 @@ public class DbTrackerTests public void TestTrackOnlyCreatedDb() { using IContainer container = new ContainerBuilder() - .AddSingleton() - .AddDecorator() + .AddSingleton() + .AddSingleton(new MetricsConfig()) + .AddSingleton(LimboLogs.Instance) + .AddSingleton(NoopMonitoringService.Instance) + .AddDecorator() .AddSingleton() .Build(); IDbFactory dbFactory = container.Resolve(); - DbTracker tracker = container.Resolve(); + DbMonitoringModule.DbTracker tracker = container.Resolve(); tracker.GetAllDbMeta().Count().Should().Be(0); dbFactory.CreateDb(new DbSettings("TestDb", "TestDb")); @@ -30,4 +47,107 @@ public void TestTrackOnlyCreatedDb() var firstEntry = tracker.GetAllDbMeta().First(); firstEntry.Key.Should().Be("TestDb"); } + + [Parallelizable(ParallelScope.None)] + [TestCase(true)] + [TestCase(false)] + public void TestUpdateDbMetric(bool isProcessing) + { + IBlockProcessingQueue queue = Substitute.For(); + (IContainer container, Action updateAction, FakeDb fakeDb) = ConfigureMetricUpdater((builder) => builder.AddSingleton(queue)); + using var _ = container; + + // Reset + Metrics.DbReads["TestDb"] = 0; + + if (isProcessing) + { + container.Resolve(); // Only setup is something requested the block processing queue. + queue.IsEmpty.Returns(false); + queue.BlockAdded += Raise.EventWith(new BlockEventArgs(Build.A.Block.TestObject)); + } + + updateAction!(); + + // Assert + Assert.That(Metrics.DbReads["TestDb"], isProcessing ? Is.EqualTo(0) : Is.EqualTo(10)); + } + + [Parallelizable(ParallelScope.None)] + [Test] + public void DoesNotUpdateIfIntervalHasNotPassed() + { + (IContainer container, Action updateAction, FakeDb fakeDb) = ConfigureMetricUpdater(); + using var _ = container; + + container.Resolve().CreateDb(new DbSettings("TestDb", "TestDb")); + + // Reset + Metrics.DbReads["TestDb"] = 0; + + updateAction!(); + Assert.That(Metrics.DbReads["TestDb"], Is.EqualTo(10)); + + fakeDb.SetMetric(new IDbMeta.DbMetric() + { + TotalReads = 11 + }); + + updateAction!(); + Assert.That(Metrics.DbReads["TestDb"], Is.EqualTo(10)); + } + + private (IContainer, Action, FakeDb) ConfigureMetricUpdater(Action? configurer = null) + { + IDbFactory fakeDbFactory = Substitute.For(); + + IMonitoringService monitoringService = Substitute.For(); + ContainerBuilder builder = new ContainerBuilder() + .AddModule(new DbModule(new InitConfig(), new ReceiptConfig(), new SyncConfig())) + .AddModule(new DbMonitoringModule()) + .AddSingleton(new MetricsConfig()) + .AddSingleton(LimboLogs.Instance) + .AddSingleton(monitoringService) + .AddDecorator() + .AddSingleton(fakeDbFactory); + + configurer?.Invoke(builder); + + IContainer container = builder + .Build(); + + IDbMeta.DbMetric metric = new IDbMeta.DbMetric() + { + TotalReads = 10 + }; + FakeDb fakeDb = new FakeDb(metric); + fakeDbFactory.CreateDb(Arg.Any()).Returns(fakeDb); + + Action updateAction = null; + monitoringService + .When((m) => m.AddMetricsUpdateAction(Arg.Any())) + .Do((c) => + { + updateAction = (Action)c[0]; + }); + + container.Resolve().CreateDb(new DbSettings("TestDb", "TestDb")); + + return (container, updateAction, fakeDb); + } + + private class FakeDb(IDbMeta.DbMetric metric) : TestMemDb, IDbMeta + { + private IDbMeta.DbMetric _metric = metric; + + public override IDbMeta.DbMetric GatherMetric(bool includeSharedCache = false) + { + return _metric; + } + + internal void SetMetric(IDbMeta.DbMetric metric) + { + _metric = metric; + } + } } diff --git a/src/Nethermind/Nethermind.Db/DbTracker.cs b/src/Nethermind/Nethermind.Db/DbTracker.cs deleted file mode 100644 index 95829f161bea..000000000000 --- a/src/Nethermind/Nethermind.Db/DbTracker.cs +++ /dev/null @@ -1,48 +0,0 @@ -// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using System; -using System.Collections.Generic; -using NonBlocking; - -namespace Nethermind.Db; - -public class DbTracker -{ - private readonly ConcurrentDictionary _createdDbs = new ConcurrentDictionary(); - - public void AddDb(string name, IDbMeta dbMeta) - { - _createdDbs.TryAdd(name, dbMeta); - } - - public IEnumerable> GetAllDbMeta() - { - return _createdDbs; - } - - public class DbFactoryInterceptor(DbTracker tracker, IDbFactory baseFactory) : IDbFactory - { - public IDb CreateDb(DbSettings dbSettings) - { - IDb db = baseFactory.CreateDb(dbSettings); - if (db is IDbMeta dbMeta) - { - tracker.AddDb(dbSettings.DbName, dbMeta); - } - return db; - } - - public IColumnsDb CreateColumnsDb(DbSettings dbSettings) where T : struct, Enum - { - IColumnsDb db = baseFactory.CreateColumnsDb(dbSettings); - if (db is IDbMeta dbMeta) - { - tracker.AddDb(dbSettings.DbName, dbMeta); - } - return db; - } - - public string GetFullDbPath(DbSettings dbSettings) => baseFactory.GetFullDbPath(dbSettings); - } -} diff --git a/src/Nethermind/Nethermind.Db/MemDb.cs b/src/Nethermind/Nethermind.Db/MemDb.cs index e4fd05660731..72e2087e3551 100644 --- a/src/Nethermind/Nethermind.Db/MemDb.cs +++ b/src/Nethermind/Nethermind.Db/MemDb.cs @@ -151,7 +151,7 @@ public virtual void Set(ReadOnlySpan key, byte[]? value, WriteFlags flags _spanDb[key] = value; } - public IDbMeta.DbMetric GatherMetric(bool includeSharedCache = false) => new() { Size = Count }; + public virtual IDbMeta.DbMetric GatherMetric(bool includeSharedCache = false) => new() { Size = Count }; private IEnumerable> OrderedDb => _db.OrderBy(kvp => kvp.Key, Bytes.Comparer); } diff --git a/src/Nethermind/Nethermind.Init/Modules/DbModule.cs b/src/Nethermind/Nethermind.Init/Modules/DbModule.cs index 8e308a7479b1..cd990ccdfe01 100644 --- a/src/Nethermind/Nethermind.Init/Modules/DbModule.cs +++ b/src/Nethermind/Nethermind.Init/Modules/DbModule.cs @@ -57,11 +57,6 @@ protected override void Load(ContainerBuilder builder) return sortedKeyValue; }) - // Monitoring use these to track active db. We intercept db factory to keep them lazy. Does not - // track db that is not created by db factory though... - .AddSingleton() - .AddDecorator() - .AddDatabase(DbNames.State) .AddDatabase(DbNames.Code) .AddDatabase(DbNames.Metadata) diff --git a/src/Nethermind/Nethermind.Init/Modules/DbMonitoringModule.cs b/src/Nethermind/Nethermind.Init/Modules/DbMonitoringModule.cs new file mode 100644 index 000000000000..e4bb8c6d8741 --- /dev/null +++ b/src/Nethermind/Nethermind.Init/Modules/DbMonitoringModule.cs @@ -0,0 +1,132 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using Autofac; +using Nethermind.Consensus.Processing; +using Nethermind.Core; +using Nethermind.Db; +using Nethermind.Logging; +using Nethermind.Monitoring; +using Nethermind.Monitoring.Config; + +namespace Nethermind.Init.Modules; + +public class DbMonitoringModule : Module +{ + protected override void Load(ContainerBuilder builder) + { + base.Load(builder); + + // Intercept created db to publish metric. + // Dont use constructor injection to get all db because that would resolve all db + // making them not lazy. + builder + .AddSingleton() + .AddDecorator() + + // Intercept block processing by checking the queue and pausing the metrics when that happen. + // Dont use constructor injection because this would prevent the metric from being updated before + // the block processing chain is constructed, eg: VerifyTrie or import jobs. + .Intercept((processingQueue, ctx) => + { + if (!ctx.Resolve().PauseDbMetricDuringBlockProcessing) return; + + // Do not update db metrics while processing a block + DbTracker updater = ctx.Resolve(); + processingQueue.BlockAdded += (sender, args) => updater.Paused = !processingQueue.IsEmpty; + processingQueue.BlockRemoved += (sender, args) => updater.Paused = !processingQueue.IsEmpty; + }) + ; + } + + public class DbTracker + { + private readonly ConcurrentDictionary _createdDbs = new ConcurrentDictionary(); + private readonly int _intervalSec; + private long _lastDbMetricsUpdate = 0; + + private ILogger _logger; + + public DbTracker(IMonitoringService monitoringService, IMetricsConfig metricsConfig, ILogManager logManager) + { + _intervalSec = metricsConfig.DbMetricIntervalSeconds; + _logger = logManager.GetClassLogger(); + + if (metricsConfig.EnableDbSizeMetrics) + { + monitoringService.AddMetricsUpdateAction(UpdateDbMetrics); + } + } + + public void AddDb(string name, IDbMeta dbMeta) + { + _createdDbs.TryAdd(name, dbMeta); + } + + public IEnumerable> GetAllDbMeta() + { + return _createdDbs; + } + + public bool Paused { get; set; } = false; + + private void UpdateDbMetrics() + { + try + { + if (Paused) return; + + if (Environment.TickCount64 - _lastDbMetricsUpdate < _intervalSec * 1000) + { + // Update based on configured interval + return; + } + + foreach (KeyValuePair kv in GetAllDbMeta()) + { + // Note: At the moment, the metric for a columns db is combined across column. + IDbMeta.DbMetric dbMetric = kv.Value.GatherMetric(includeSharedCache: kv.Key == DbNames.State); // Only include shared cache if state db + Db.Metrics.DbSize[kv.Key] = dbMetric.Size; + Db.Metrics.DbBlockCacheSize[kv.Key] = dbMetric.CacheSize; + Db.Metrics.DbMemtableSize[kv.Key] = dbMetric.MemtableSize; + Db.Metrics.DbIndexFilterSize[kv.Key] = dbMetric.IndexSize; + Db.Metrics.DbReads[kv.Key] = dbMetric.TotalReads; + Db.Metrics.DbWrites[kv.Key] = dbMetric.TotalWrites; + } + _lastDbMetricsUpdate = Environment.TickCount64; + } + catch (Exception e) + { + if (_logger.IsError) _logger.Error("Error during updating db metrics", e); + } + } + + public class DbFactoryInterceptor(DbTracker tracker, IDbFactory baseFactory) : IDbFactory + { + public IDb CreateDb(DbSettings dbSettings) + { + IDb db = baseFactory.CreateDb(dbSettings); + if (db is IDbMeta dbMeta) + { + tracker.AddDb(dbSettings.DbName, dbMeta); + } + return db; + } + + public IColumnsDb CreateColumnsDb(DbSettings dbSettings) where T : struct, Enum + { + IColumnsDb db = baseFactory.CreateColumnsDb(dbSettings); + if (db is IDbMeta dbMeta) + { + tracker.AddDb(dbSettings.DbName, dbMeta); + } + return db; + } + + public string GetFullDbPath(DbSettings dbSettings) => baseFactory.GetFullDbPath(dbSettings); + } + } +} diff --git a/src/Nethermind/Nethermind.Init/Modules/MonitoringModule.cs b/src/Nethermind/Nethermind.Init/Modules/MonitoringModule.cs new file mode 100644 index 000000000000..814ab35de125 --- /dev/null +++ b/src/Nethermind/Nethermind.Init/Modules/MonitoringModule.cs @@ -0,0 +1,119 @@ +// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Collections.Generic; +using System.Linq; +using Autofac; +using DotNetty.Buffers; +using Nethermind.Blockchain.Synchronization; +using Nethermind.Core; +using Nethermind.Core.Specs; +using Nethermind.Db; +using Nethermind.Facade.Eth; +using Nethermind.Logging; +using Nethermind.Monitoring; +using Nethermind.Monitoring.Config; +using Nethermind.Monitoring.Metrics; +using Nethermind.Serialization.Rlp; + +namespace Nethermind.Init.Modules; + +public class MonitoringModule(IMetricsConfig metricsConfig) : Module +{ + protected override void Load(ContainerBuilder builder) + { + if (metricsConfig.Enabled || metricsConfig.CountersEnabled) + { + builder + .AddSingleton() + .AddSingleton( + PrepareProductInfoMetrics) + + .AddSingleton() + .Intercept(ConfigureDefaultMetrics) + + .Intercept((syncInfo, ctx) => + { + ctx.Resolve().AddMetricsUpdateAction(() => + { + Synchronization.Metrics.SyncTime = (long?)syncInfo.UpdateAndGetSyncTime().TotalSeconds ?? 0; + }); + }) + + ; + } + else + { + builder.AddSingleton(); + } + } + + private IMetricsController PrepareProductInfoMetrics(IMetricsConfig metricsConfig, ISyncConfig syncConfig, IPruningConfig pruningConfig, ISpecProvider specProvider) + { + // Need to be set here, or we cant start before blocktree, which is the one that set this normally. + ProductInfo.Network = $"{(specProvider.ChainId == specProvider.NetworkId ? BlockchainIds.GetBlockchainName(specProvider.NetworkId) : specProvider.ChainId)}"; + + ProductInfo.Instance = metricsConfig.NodeName; + + ProductInfo.SyncType = syncConfig.FastSync + ? syncConfig.SnapSync ? "Snap" : "Fast" + : "Full"; + + + ProductInfo.PruningMode = pruningConfig.Mode.ToString(); + Metrics.Version = VersionToMetrics.ConvertToNumber(ProductInfo.Version); + + IMetricsController controller = new MetricsController(metricsConfig); + + IEnumerable metrics = TypeDiscovery.FindNethermindBasedTypes(nameof(Metrics)); + foreach (Type metric in metrics) + { + controller.RegisterMetrics(metric); + } + + return controller; + } + + private void ConfigureDefaultMetrics(IMonitoringService monitoringService, IComponentContext ctx) + { + // Note: Do not add dependencies outside of monitoring module. + AllocatorMetricsUpdater allocatorMetricsUpdater = ctx.Resolve(); + monitoringService.AddMetricsUpdateAction(() => allocatorMetricsUpdater.UpdateAllocatorMetrics()); + } + + private class AllocatorMetricsUpdater(ILogManager logManager) + { + ILogger _logger = logManager.GetClassLogger(); + + public void UpdateAllocatorMetrics() + { + try + { + SetAllocatorMetrics(NethermindBuffers.RlpxAllocator, "rlpx"); + SetAllocatorMetrics(NethermindBuffers.DiscoveryAllocator, "discovery"); + SetAllocatorMetrics(NethermindBuffers.Default, "default"); + SetAllocatorMetrics(PooledByteBufferAllocator.Default, "netty_default"); + } + catch (Exception e) + { + if (_logger.IsError) _logger.Error("Error during allocator metrics", e); + } + } + + private static void SetAllocatorMetrics(IByteBufferAllocator allocator, string name) + { + if (allocator is PooledByteBufferAllocator byteBufferAllocator) + { + PooledByteBufferAllocatorMetric metric = byteBufferAllocator.Metric; + Serialization.Rlp.Metrics.AllocatorArenaCount[name] = metric.DirectArenas().Count; + Serialization.Rlp.Metrics.AllocatorChunkSize[name] = metric.ChunkSize; + Serialization.Rlp.Metrics.AllocatorUsedHeapMemory[name] = metric.UsedHeapMemory; + Serialization.Rlp.Metrics.AllocatorUsedDirectMemory[name] = metric.UsedDirectMemory; + Serialization.Rlp.Metrics.AllocatorActiveAllocations[name] = metric.HeapArenas().Sum((it) => it.NumActiveAllocations); + Serialization.Rlp.Metrics.AllocatorActiveAllocationBytes[name] = metric.HeapArenas().Sum((it) => it.NumActiveBytes); + Serialization.Rlp.Metrics.AllocatorAllocations[name] = metric.HeapArenas().Sum((it) => it.NumAllocations); + } + } + } +} diff --git a/src/Nethermind/Nethermind.Init/Modules/NethermindModule.cs b/src/Nethermind/Nethermind.Init/Modules/NethermindModule.cs index 9ff2ea05f82f..41df70db2387 100644 --- a/src/Nethermind/Nethermind.Init/Modules/NethermindModule.cs +++ b/src/Nethermind/Nethermind.Init/Modules/NethermindModule.cs @@ -6,7 +6,6 @@ using Nethermind.Abi; using Nethermind.Api; using Nethermind.Blockchain; -using Nethermind.Blockchain.Find; using Nethermind.Blockchain.Receipts; using Nethermind.Blockchain.Spec; using Nethermind.Blockchain.Synchronization; @@ -16,11 +15,10 @@ using Nethermind.Core.Specs; using Nethermind.Core.Timers; using Nethermind.Crypto; -using Nethermind.Db; using Nethermind.Era1; -using Nethermind.History; using Nethermind.JsonRpc; using Nethermind.Logging; +using Nethermind.Monitoring.Config; using Nethermind.Network.Config; using Nethermind.Runner.Ethereum.Modules; using Nethermind.Specs.ChainSpecStyle; @@ -48,6 +46,7 @@ protected override void Load(ContainerBuilder builder) configProvider.GetConfig(), configProvider.GetConfig() )) + .AddModule(new DbMonitoringModule()) .AddModule(new WorldStateModule(configProvider.GetConfig())) .AddModule(new PrewarmerModule(configProvider.GetConfig())) .AddModule(new BuiltInStepsModule()) @@ -56,6 +55,7 @@ protected override void Load(ContainerBuilder builder) .AddSource(new ConfigRegistrationSource()) .AddModule(new BlockProcessingModule(configProvider.GetConfig(), configProvider.GetConfig())) .AddModule(new BlockTreeModule(configProvider.GetConfig())) + .AddModule(new MonitoringModule(configProvider.GetConfig())) .AddSingleton() .AddKeyedSingleton(IProtectedPrivateKey.NodeKey, (ctx) => ctx.Resolve().NodeKey!) diff --git a/src/Nethermind/Nethermind.Init/Modules/RpcModules.cs b/src/Nethermind/Nethermind.Init/Modules/RpcModules.cs index 8f0ee78e45ff..c9ff690ad826 100644 --- a/src/Nethermind/Nethermind.Init/Modules/RpcModules.cs +++ b/src/Nethermind/Nethermind.Init/Modules/RpcModules.cs @@ -8,7 +8,6 @@ using Nethermind.Blockchain.Filters; using Nethermind.Blockchain.Receipts; using Nethermind.Config; -using Nethermind.Consensus.Processing; using Nethermind.Consensus.Tracing; using Nethermind.Core; using Nethermind.Core.Timers; @@ -31,7 +30,6 @@ using Nethermind.JsonRpc.Modules.Trace; using Nethermind.JsonRpc.Modules.TxPool; using Nethermind.JsonRpc.Modules.Web3; -using Nethermind.Logging; using Nethermind.Network; using Nethermind.Network.Config; using Nethermind.Sockets; diff --git a/src/Nethermind/Nethermind.Init/Steps/EthereumStepsManager.cs b/src/Nethermind/Nethermind.Init/Steps/EthereumStepsManager.cs index a96786ba0943..2a12b6221096 100644 --- a/src/Nethermind/Nethermind.Init/Steps/EthereumStepsManager.cs +++ b/src/Nethermind/Nethermind.Init/Steps/EthereumStepsManager.cs @@ -254,10 +254,7 @@ private string BuildStepDependencyTree(Dictionary stepInfoMap List deps = deduplicatedDependency[node]; sb.Append(dependentsMap[node].Count == 0 ? "● " : "○ "); sb.Append(node); - if (deps.Count != 0) - { - sb.AppendLine($" (depends on {string.Join(", ", deps)})"); - } + sb.AppendLine(deps.Count != 0 ? $" (depends on {string.Join(", ", deps)})" : ""); } return sb.ToString(); diff --git a/src/Nethermind/Nethermind.Init/Steps/StartMonitoring.cs b/src/Nethermind/Nethermind.Init/Steps/StartMonitoring.cs index 9f4026090f61..a3ac17f3ea7a 100644 --- a/src/Nethermind/Nethermind.Init/Steps/StartMonitoring.cs +++ b/src/Nethermind/Nethermind.Init/Steps/StartMonitoring.cs @@ -1,38 +1,20 @@ // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only -using System; -using System.Collections.Generic; -using System.Linq; using System.Threading; using System.Threading.Tasks; -using DotNetty.Buffers; using Nethermind.Api.Steps; -using Nethermind.Blockchain; -using Nethermind.Blockchain.Synchronization; -using Nethermind.Core; -using Nethermind.Core.ServiceStopper; -using Nethermind.Db; -using Nethermind.Facade.Eth; using Nethermind.Logging; using Nethermind.Monitoring; using Nethermind.Monitoring.Config; -using Nethermind.Monitoring.Metrics; -using Nethermind.Serialization.Rlp; -using Type = System.Type; namespace Nethermind.Init.Steps; -[RunnerStepDependencies(typeof(InitializeBlockchain))] +[RunnerStepDependencies()] public class StartMonitoring( - IEthSyncingInfo ethSyncingInfo, - DbTracker dbTracker, - IPruningConfig pruningConfig, - ISyncConfig syncConfig, - IServiceStopper serviceStopper, + IMonitoringService monitoringService, ILogManager logManager, - IMetricsConfig metricsConfig, - ChainHeadInfoProvider chainHeadInfoProvider + IMetricsConfig metricsConfig ) : IStep { private readonly ILogger _logger = logManager.GetClassLogger(); @@ -45,32 +27,13 @@ public async Task Execute(CancellationToken cancellationToken) logManager.SetGlobalVariable("nodeName", metricsConfig.NodeName); } - MetricsController? controller = null; - if (metricsConfig.Enabled || metricsConfig.CountersEnabled) - { - PrepareProductInfoMetrics(); - controller = new(metricsConfig); - - IEnumerable metrics = TypeDiscovery.FindNethermindBasedTypes(nameof(Metrics)); - foreach (Type metric in metrics) - { - controller.RegisterMetrics(metric); - } - } - if (metricsConfig.Enabled) { - MonitoringService monitoringService = new(controller, metricsConfig, logManager); - - SetupMetrics(monitoringService); - await monitoringService.StartAsync().ContinueWith(x => { if (x.IsFaulted && _logger.IsError) _logger.Error("Error during starting a monitoring.", x.Exception); }, cancellationToken); - - serviceStopper.AddStoppable(monitoringService); } else { @@ -86,125 +49,5 @@ await monitoringService.StartAsync().ContinueWith(x => } } - private void SetupMetrics(MonitoringService monitoringService) - { - if (metricsConfig.EnableDbSizeMetrics) - { - monitoringService.AddMetricsUpdateAction(() => Task.Run(() => UpdateDbMetrics())); - } - - if (metricsConfig.EnableDetailedMetric) - { - monitoringService.AddMetricsUpdateAction(() => Task.Run(() => UpdateAllocatorMetrics())); - } - - monitoringService.AddMetricsUpdateAction(() => - { - Synchronization.Metrics.SyncTime = (long?)ethSyncingInfo?.UpdateAndGetSyncTime().TotalSeconds ?? 0; - }); - } - - private bool _isUpdatingDbMetrics = false; - private long _lastDbMetricsUpdate = 0; - private void UpdateDbMetrics() - { - if (!Interlocked.Exchange(ref _isUpdatingDbMetrics, true)) - { - try - { - if (Environment.TickCount64 - _lastDbMetricsUpdate < 60_000) - { - // Update max every minute - return; - } - if (chainHeadInfoProvider.IsProcessingBlock) - { - // Do not update db metrics while processing a block - return; - } - - foreach (KeyValuePair kv in dbTracker.GetAllDbMeta()) - { - // Note: At the moment, the metric for a columns db is combined across column. - IDbMeta.DbMetric dbMetric = kv.Value.GatherMetric(includeSharedCache: kv.Key == DbNames.State); // Only include shared cache if state db - Db.Metrics.DbSize[kv.Key] = dbMetric.Size; - Db.Metrics.DbBlockCacheSize[kv.Key] = dbMetric.CacheSize; - Db.Metrics.DbMemtableSize[kv.Key] = dbMetric.MemtableSize; - Db.Metrics.DbIndexFilterSize[kv.Key] = dbMetric.IndexSize; - Db.Metrics.DbReads[kv.Key] = dbMetric.TotalReads; - Db.Metrics.DbWrites[kv.Key] = dbMetric.TotalWrites; - } - _lastDbMetricsUpdate = Environment.TickCount64; - } - catch (Exception e) - { - if (_logger.IsError) _logger.Error("Error during updating db metrics", e); - } - finally - { - Volatile.Write(ref _isUpdatingDbMetrics, false); - } - } - } - - private bool _isUpdatingAllocatorMetrics = false; - private void UpdateAllocatorMetrics() - { - if (!Interlocked.Exchange(ref _isUpdatingAllocatorMetrics, true)) - { - try - { - SetAllocatorMetrics(NethermindBuffers.RlpxAllocator, "rlpx"); - SetAllocatorMetrics(NethermindBuffers.DiscoveryAllocator, "discovery"); - SetAllocatorMetrics(NethermindBuffers.Default, "default"); - SetAllocatorMetrics(PooledByteBufferAllocator.Default, "netty_default"); - } - catch (Exception e) - { - if (_logger.IsError) _logger.Error("Error during allocator metrics", e); - } - finally - { - Volatile.Write(ref _isUpdatingAllocatorMetrics, false); - } - } - } - - private static void SetAllocatorMetrics(IByteBufferAllocator allocator, string name) - { - if (allocator is PooledByteBufferAllocator byteBufferAllocator) - { - PooledByteBufferAllocatorMetric metric = byteBufferAllocator.Metric; - Serialization.Rlp.Metrics.AllocatorArenaCount[name] = metric.DirectArenas().Count; - Serialization.Rlp.Metrics.AllocatorChunkSize[name] = metric.ChunkSize; - Serialization.Rlp.Metrics.AllocatorUsedHeapMemory[name] = metric.UsedHeapMemory; - Serialization.Rlp.Metrics.AllocatorUsedDirectMemory[name] = metric.UsedDirectMemory; - Serialization.Rlp.Metrics.AllocatorActiveAllocations[name] = metric.HeapArenas().Sum((it) => it.NumActiveAllocations); - Serialization.Rlp.Metrics.AllocatorActiveAllocationBytes[name] = metric.HeapArenas().Sum((it) => it.NumActiveBytes); - Serialization.Rlp.Metrics.AllocatorAllocations[name] = metric.HeapArenas().Sum((it) => it.NumAllocations); - } - } - - private void PrepareProductInfoMetrics() - { - ProductInfo.Instance = metricsConfig.NodeName; - - if (syncConfig.SnapSync) - { - ProductInfo.SyncType = "Snap"; - } - else if (syncConfig.FastSync) - { - ProductInfo.SyncType = "Fast"; - } - else - { - ProductInfo.SyncType = "Full"; - } - - ProductInfo.PruningMode = pruningConfig.Mode.ToString(); - Metrics.Version = VersionToMetrics.ConvertToNumber(ProductInfo.Version); - } - public bool MustInitialize => false; } diff --git a/src/Nethermind/Nethermind.Monitoring/Config/IMetricsConfig.cs b/src/Nethermind/Nethermind.Monitoring/Config/IMetricsConfig.cs index 83ac5bcd7e50..0eb30fade851 100644 --- a/src/Nethermind/Nethermind.Monitoring/Config/IMetricsConfig.cs +++ b/src/Nethermind/Nethermind.Monitoring/Config/IMetricsConfig.cs @@ -26,6 +26,12 @@ public interface IMetricsConfig : IConfig [ConfigItem(DefaultValue = "5", Description = "The frequency of pushing metrics to Prometheus, in seconds.")] int IntervalSeconds { get; } + [ConfigItem(DefaultValue = "60", Description = "The frequency of updating db metrics, in seconds.")] + int DbMetricIntervalSeconds { get; } + + [ConfigItem(DefaultValue = "true", Description = "Pause db metric collection during block processing to prevent overhead.")] + bool PauseDbMetricDuringBlockProcessing { get; } + [ConfigItem(Description = "The name to display on the Grafana dashboard.", DefaultValue = "Nethermind")] string NodeName { get; } diff --git a/src/Nethermind/Nethermind.Monitoring/Config/MetricsConfig.cs b/src/Nethermind/Nethermind.Monitoring/Config/MetricsConfig.cs index 77e194cc7f85..c9dc00e6f7df 100644 --- a/src/Nethermind/Nethermind.Monitoring/Config/MetricsConfig.cs +++ b/src/Nethermind/Nethermind.Monitoring/Config/MetricsConfig.cs @@ -11,6 +11,8 @@ public class MetricsConfig : IMetricsConfig public bool CountersEnabled { get; set; } = false; public string PushGatewayUrl { get; set; } = null; public int IntervalSeconds { get; set; } = 5; + public int DbMetricIntervalSeconds { get; set; } = 60; + public bool PauseDbMetricDuringBlockProcessing { get; set; } = true; public string NodeName { get; set; } = "Nethermind"; public bool EnableDbSizeMetrics { get; set; } = true; public string MonitoringGroup { get; set; } = "nethermind"; diff --git a/src/Nethermind/Nethermind.Monitoring/IMonitoringService .cs b/src/Nethermind/Nethermind.Monitoring/IMonitoringService.cs similarity index 92% rename from src/Nethermind/Nethermind.Monitoring/IMonitoringService .cs rename to src/Nethermind/Nethermind.Monitoring/IMonitoringService.cs index f8d13fc417f4..d023c62503a7 100644 --- a/src/Nethermind/Nethermind.Monitoring/IMonitoringService .cs +++ b/src/Nethermind/Nethermind.Monitoring/IMonitoringService.cs @@ -9,7 +9,6 @@ namespace Nethermind.Monitoring public interface IMonitoringService { Task StartAsync(); - Task StopAsync(); void AddMetricsUpdateAction(Action callback); } } diff --git a/src/Nethermind/Nethermind.Monitoring/Metrics/IMetricsController.cs b/src/Nethermind/Nethermind.Monitoring/Metrics/IMetricsController.cs index a5ed5783c919..e9a2ab9ee212 100644 --- a/src/Nethermind/Nethermind.Monitoring/Metrics/IMetricsController.cs +++ b/src/Nethermind/Nethermind.Monitoring/Metrics/IMetricsController.cs @@ -2,14 +2,15 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; +using System.Threading; +using System.Threading.Tasks; namespace Nethermind.Monitoring.Metrics { public interface IMetricsController { void RegisterMetrics(Type type); - void StartUpdating(); - void StopUpdating(); + Task RunTimer(CancellationToken cancellationToken); void AddMetricsUpdateAction(Action callback); } } diff --git a/src/Nethermind/Nethermind.Monitoring/Metrics/MetricsController.cs b/src/Nethermind/Nethermind.Monitoring/Metrics/MetricsController.cs index e8b139e4f4f6..4fa90cc8b97e 100644 --- a/src/Nethermind/Nethermind.Monitoring/Metrics/MetricsController.cs +++ b/src/Nethermind/Nethermind.Monitoring/Metrics/MetricsController.cs @@ -14,6 +14,7 @@ using System.Runtime.Serialization; using System.Text.RegularExpressions; using System.Threading; +using System.Threading.Tasks; using Nethermind.Core; using Nethermind.Core.Attributes; using Nethermind.Core.Collections; @@ -29,7 +30,6 @@ namespace Nethermind.Monitoring.Metrics public partial class MetricsController : IMetricsController { private readonly int _intervalMilliseconds; - private Timer _timer = null!; private static bool _staticLabelsInitialized; private readonly Dictionary _metricUpdaters = new(); @@ -326,29 +326,23 @@ public MetricsController(IMetricsConfig metricsConfig) _enableDetailedMetric = metricsConfig.EnableDetailedMetric; } - public void StartUpdating() => _timer = new Timer(UpdateAllMetrics, null, TimeSpan.Zero, TimeSpan.FromMilliseconds(_intervalMilliseconds)); - - public void StopUpdating() => _timer?.Change(Timeout.Infinite, 0); - - private void UpdateAllMetrics(object? state) => UpdateAllMetrics(); - - private bool _isUpdating = false; - public void UpdateAllMetrics() + public async Task RunTimer(CancellationToken cancellationToken) { - if (!Interlocked.Exchange(ref _isUpdating, true)) + using var standardTimer = new PeriodicTimer(TimeSpan.FromMilliseconds(_intervalMilliseconds)); + + try { - try + while (await standardTimer.WaitForNextTickAsync(cancellationToken)) { - UpdateAllMetricsInner(); - } - finally - { - Volatile.Write(ref _isUpdating, false); + UpdateAllMetrics(); } } + catch (OperationCanceledException) + { + } } - private void UpdateAllMetricsInner() + public void UpdateAllMetrics() { foreach (Action callback in _callbacks) { diff --git a/src/Nethermind/Nethermind.Monitoring/MonitoringService.cs b/src/Nethermind/Nethermind.Monitoring/MonitoringService.cs index 1ee91ac12dba..1537447a0442 100644 --- a/src/Nethermind/Nethermind.Monitoring/MonitoringService.cs +++ b/src/Nethermind/Nethermind.Monitoring/MonitoringService.cs @@ -8,12 +8,12 @@ using Nethermind.Monitoring.Metrics; using Nethermind.Monitoring.Config; using System.Net.Sockets; -using Nethermind.Core.ServiceStopper; +using System.Threading; using Prometheus; namespace Nethermind.Monitoring; -public class MonitoringService : IMonitoringService, IStoppableService +public class MonitoringService : IMonitoringService, IAsyncDisposable { private readonly IMetricsController _metricsController; private readonly ILogger _logger; @@ -25,9 +25,18 @@ public class MonitoringService : IMonitoringService, IStoppableService private readonly bool _pushEnabled; private readonly string _pushGatewayUrl; private readonly int _intervalSeconds; + private readonly CancellationTokenSource _timerCancellationSource; - public MonitoringService(IMetricsController metricsController, IMetricsConfig metricsConfig, ILogManager logManager) + private Task _monitoringTimerTask = Task.CompletedTask; + private int _isDisposed = 0; + + public MonitoringService( + IMetricsController metricsController, + IMetricsConfig metricsConfig, + ILogManager logManager + ) { + _timerCancellationSource = new CancellationTokenSource(); _metricsController = metricsController ?? throw new ArgumentNullException(nameof(metricsController)); string exposeHost = metricsConfig.ExposeHost; @@ -85,7 +94,17 @@ public Task StartAsync() new NethermindKestrelMetricServer(_exposeHost, _exposePort.Value).Start(); } - _metricsController.StartUpdating(); + _monitoringTimerTask = Task.Run(async () => + { + try + { + await _metricsController.RunTimer(_timerCancellationSource.Token); + } + catch (Exception ex) + { + if (_logger.IsError) _logger.Error($"Monitoring timer failed: {ex}"); + } + }); if (_logger.IsInfo) _logger.Info($"Started monitoring for the group: {_options.Group}, instance: {_options.Instance}"); return Task.CompletedTask; @@ -96,13 +115,6 @@ public void AddMetricsUpdateAction(Action callback) _metricsController.AddMetricsUpdateAction(callback); } - public Task StopAsync() - { - _metricsController.StopUpdating(); - - return Task.CompletedTask; - } - public string Description => "Monitoring service"; private Options GetOptions(IMetricsConfig config) @@ -121,4 +133,12 @@ private class Options(string job, string group, string instance) public string Instance { get; } = instance; public string Group { get; } = group; } + + public async ValueTask DisposeAsync() + { + if (Interlocked.CompareExchange(ref _isDisposed, 1, 0) != 0) return; + await _timerCancellationSource.CancelAsync(); + await _monitoringTimerTask; + _timerCancellationSource.Dispose(); + } } diff --git a/src/Nethermind/Nethermind.Monitoring/NoopMonitoringService.cs b/src/Nethermind/Nethermind.Monitoring/NoopMonitoringService.cs new file mode 100644 index 000000000000..9f7fc17d71c9 --- /dev/null +++ b/src/Nethermind/Nethermind.Monitoring/NoopMonitoringService.cs @@ -0,0 +1,22 @@ +// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Threading.Tasks; +using Autofac; + +namespace Nethermind.Monitoring; + +public class NoopMonitoringService : IMonitoringService +{ + public static IMonitoringService Instance = new NoopMonitoringService(); + + public Task StartAsync() + { + return Task.CompletedTask; + } + + public void AddMetricsUpdateAction(Action callback) + { + } +}