From 2a429de74ab97196eaef7dd0d8c53e7cc009493b Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 26 May 2026 09:50:21 -0700 Subject: [PATCH 1/7] fix(runtime): defer ClientDirectory background loop to BecomeActive lifecycle stage Defer the ClientDirectory background loop from RuntimeGrainServices to BecomeActive so it does not start running until the silo is ready to host activations. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/Orleans.Runtime/GrainDirectory/ClientDirectory.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Orleans.Runtime/GrainDirectory/ClientDirectory.cs b/src/Orleans.Runtime/GrainDirectory/ClientDirectory.cs index 3115ee0e01..e55a62d1af 100644 --- a/src/Orleans.Runtime/GrainDirectory/ClientDirectory.cs +++ b/src/Orleans.Runtime/GrainDirectory/ClientDirectory.cs @@ -522,7 +522,7 @@ void ILifecycleParticipant.Participate(ISiloLifecycle lifecycle) { lifecycle.Subscribe( nameof(ClientDirectory), - ServiceLifecycleStage.RuntimeGrainServices, + ServiceLifecycleStage.BecomeActive, StartPublishingRoutingTable, StopPublishingRoutingTable); From 9d547a573d87e554c283ebb08c00e1415cbb0bba Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 26 May 2026 21:03:34 -0700 Subject: [PATCH 2/7] feat(runtime): add silo catalog and directory activation metrics Add an activation-duration histogram tagged by activation status and directory enablement, register an activation-registration-failed counter, and emit directory-error/duplicate/canceled/success statuses from the activation lifecycle. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Diagnostics/Metrics/CatalogInstruments.cs | 57 +++++++++++-------- .../Metrics/DirectoryInstruments.cs | 44 +++++++++++--- .../Diagnostics/Metrics/InstrumentNames.cs | 4 +- src/Orleans.Runtime/Catalog/ActivationData.cs | 43 +++++++++----- .../GrainDirectory/GrainLocator.cs | 41 ++++++++++++- .../Runtime/CatalogInstrumentsTests.cs | 22 +++---- 6 files changed, 155 insertions(+), 56 deletions(-) diff --git a/src/Orleans.Core/Diagnostics/Metrics/CatalogInstruments.cs b/src/Orleans.Core/Diagnostics/Metrics/CatalogInstruments.cs index 95a6451fc8..29bf484b78 100644 --- a/src/Orleans.Core/Diagnostics/Metrics/CatalogInstruments.cs +++ b/src/Orleans.Core/Diagnostics/Metrics/CatalogInstruments.cs @@ -2,15 +2,21 @@ using System.Collections.Generic; using System.Diagnostics.Metrics; -#nullable disable namespace Orleans.Runtime; internal static class CatalogInstruments { - internal const string ActivationOutcomeCanceled = "canceled"; - internal const string ActivationOutcomeDuplicate = "duplicate"; - internal const string ActivationOutcomeFailure = "failure"; - internal const string ActivationOutcomeSuccess = "success"; + private const string MillisecondsUnit = "ms"; + private const string StatusTagName = "status"; + private const string DirectoryTagName = "directory"; + private const string DirectoryEnabled = "enabled"; + private const string DirectoryDisabled = "disabled"; + + internal const string ActivationStatusSuccess = "success"; + internal const string ActivationStatusCanceled = "canceled"; + internal const string ActivationStatusDirectoryError = "directory_error"; + internal const string ActivationStatusDuplicate = "duplicate"; + internal const string ActivationStatusError = "error"; internal const string DeactivationViaCollection = "collection"; internal const string DeactivationViaDeactivateOnIdle = "deactivateOnIdle"; @@ -24,27 +30,18 @@ internal static class CatalogInstruments internal static Counter ActivationShutdown = Instruments.Meter.CreateCounter(InstrumentNames.CATALOG_ACTIVATION_SHUTDOWN); - internal static void ActivationShutdownViaCollection() => ActivationShutdown.Add(1, new KeyValuePair("via", DeactivationViaCollection)); - internal static void ActivationShutdownViaDeactivateOnIdle() => ActivationShutdown.Add(1, new KeyValuePair("via", DeactivationViaDeactivateOnIdle)); - internal static void ActivationShutdownViaMigration() => ActivationShutdown.Add(1, new KeyValuePair("via", DeactivationViaMigration)); - internal static void ActivationShutdownViaDeactivateStuckActivation() => ActivationShutdown.Add(1, new KeyValuePair("via", DeactivationViaDeactivateStuckActivation)); - - internal static Histogram ActivationLatency = Instruments.Meter.CreateHistogram(InstrumentNames.CATALOG_ACTIVATION_LATENCY, "ms"); - internal static Histogram DeactivationLatency = Instruments.Meter.CreateHistogram(InstrumentNames.CATALOG_DEACTIVATION_LATENCY, "ms"); + internal static void ActivationShutdownViaCollection() => ActivationShutdown.Add(1, new KeyValuePair("via", DeactivationViaCollection)); + internal static void ActivationShutdownViaDeactivateOnIdle() => ActivationShutdown.Add(1, new KeyValuePair("via", DeactivationViaDeactivateOnIdle)); + internal static void ActivationShutdownViaMigration() => ActivationShutdown.Add(1, new KeyValuePair("via", DeactivationViaMigration)); + internal static void ActivationShutdownViaDeactivateStuckActivation() => ActivationShutdown.Add(1, new KeyValuePair("via", DeactivationViaDeactivateStuckActivation)); - internal static void OnActivationCompleted(TimeSpan latency, string outcome) - { - if (ActivationLatency.Enabled) - { - ActivationLatency.Record(latency.TotalMilliseconds, new KeyValuePair("outcome", outcome)); - } - } + internal static Histogram DeactivationLatency = Instruments.Meter.CreateHistogram(InstrumentNames.CATALOG_DEACTIVATION_LATENCY, MillisecondsUnit); internal static void OnDeactivationCompleted(TimeSpan latency, string via) { if (DeactivationLatency.Enabled) { - DeactivationLatency.Record(latency.TotalMilliseconds, new KeyValuePair("via", via)); + DeactivationLatency.Record(latency.TotalMilliseconds, new KeyValuePair("via", via)); } } @@ -54,17 +51,31 @@ internal static void OnDeactivationCompleted(TimeSpan latency, string via) internal static readonly Counter ActivationsCreated = Instruments.Meter.CreateCounter(InstrumentNames.CATALOG_ACTIVATION_CREATED); internal static readonly Counter ActivationsDestroyed = Instruments.Meter.CreateCounter(InstrumentNames.CATALOG_ACTIVATION_DESTROYED); + private static readonly Histogram ActivationDuration = Instruments.Meter.CreateHistogram(InstrumentNames.CATALOG_ACTIVATION_DURATION, MillisecondsUnit); + + internal static ObservableGauge? ActivationCount; - internal static ObservableGauge ActivationCount; - internal static void RegisterActivationCountObserve(Func observeValue) { ActivationCount = Instruments.Meter.CreateObservableGauge(InstrumentNames.CATALOG_ACTIVATION_COUNT, observeValue); } - internal static ObservableGauge ActivationWorkingSet; + internal static ObservableGauge? ActivationWorkingSet; internal static void RegisterActivationWorkingSetObserve(Func observeValue) { ActivationWorkingSet = Instruments.Meter.CreateObservableGauge(InstrumentNames.CATALOG_ACTIVATION_WORKING_SET, observeValue); } + + internal static void OnActivationCompleted(TimeSpan latency, string status, bool usesDirectory) + { + if (ActivationDuration.Enabled) + { + ActivationDuration.Record( + Math.Max(0, latency.TotalMilliseconds), + [ + new KeyValuePair(StatusTagName, status), + new KeyValuePair(DirectoryTagName, usesDirectory ? DirectoryEnabled : DirectoryDisabled) + ]); + } + } } diff --git a/src/Orleans.Core/Diagnostics/Metrics/DirectoryInstruments.cs b/src/Orleans.Core/Diagnostics/Metrics/DirectoryInstruments.cs index 3028827f8b..053e848107 100644 --- a/src/Orleans.Core/Diagnostics/Metrics/DirectoryInstruments.cs +++ b/src/Orleans.Core/Diagnostics/Metrics/DirectoryInstruments.cs @@ -1,13 +1,21 @@ using System; +using System.Collections.Generic; using System.Collections.Immutable; using System.Diagnostics.Metrics; using System.Threading; -#nullable disable namespace Orleans.Runtime; internal static class DirectoryInstruments { + private const string MillisecondsUnit = "ms"; + private const string StatusTagName = "status"; + private const string LocatorTagName = "locator"; + + internal const string RegistrationStatusSuccess = "success"; + internal const string RegistrationStatusCanceled = "canceled"; + internal const string RegistrationStatusError = "error"; + private static ImmutableArray CacheSizeObservers = []; internal static readonly Counter LookupsLocalIssued = Instruments.Meter.CreateCounter(InstrumentNames.DIRECTORY_LOOKUPS_LOCAL_ISSUED); @@ -31,7 +39,7 @@ internal static class DirectoryInstruments internal static readonly Histogram RangeRecoveryDuration = Instruments.Meter.CreateHistogram(InstrumentNames.DIRECTORY_RANGE_RECOVERY_DURATION); internal static readonly Histogram RangeLockHeldDuration = Instruments.Meter.CreateHistogram(InstrumentNames.DIRECTORY_RANGE_LOCK_HELD_DURATION); - internal static ObservableGauge DirectoryPartitionSize; + internal static ObservableGauge? DirectoryPartitionSize; internal static void RegisterDirectoryPartitionSizeObserve(Func observeValue) { DirectoryPartitionSize = Instruments.Meter.CreateObservableGauge(InstrumentNames.DIRECTORY_PARTITION_SIZE, observeValue); @@ -47,25 +55,25 @@ internal static IDisposable RegisterCacheSizeObserve(Func observeValue) return registration; } - internal static ObservableGauge RingSize; + internal static ObservableGauge? RingSize; internal static void RegisterRingSizeObserve(Func observeValue) { RingSize = Instruments.Meter.CreateObservableGauge(InstrumentNames.DIRECTORY_RING_RINGSIZE, observeValue); } - internal static ObservableGauge MyPortionRingDistance; + internal static ObservableGauge? MyPortionRingDistance; internal static void RegisterMyPortionRingDistanceObserve(Func observeValue) { MyPortionRingDistance = Instruments.Meter.CreateObservableGauge(InstrumentNames.DIRECTORY_RING_MYPORTION_RINGDISTANCE, observeValue); } - internal static ObservableGauge MyPortionRingPercentage; + internal static ObservableGauge? MyPortionRingPercentage; internal static void RegisterMyPortionRingPercentageObserve(Func observeValue) { MyPortionRingPercentage = Instruments.Meter.CreateObservableGauge(InstrumentNames.DIRECTORY_RING_MYPORTION_RINGPERCENTAGE, observeValue); } - internal static ObservableGauge MyPortionAverageRingPercentage; + internal static ObservableGauge? MyPortionAverageRingPercentage; internal static void RegisterMyPortionAverageRingPercentageObserve(Func observeValue) { MyPortionAverageRingPercentage = Instruments.Meter.CreateObservableGauge(InstrumentNames.DIRECTORY_RING_MYPORTION_AVERAGERINGPERCENTAGE, observeValue); @@ -75,6 +83,8 @@ internal static void RegisterMyPortionAverageRingPercentageObserve(Func o internal static readonly Counter RegistrationsSingleActLocal = Instruments.Meter.CreateCounter(InstrumentNames.DIRECTORY_REGISTRATIONS_SINGLE_ACT_LOCAL); internal static readonly Counter RegistrationsSingleActRemoteSent = Instruments.Meter.CreateCounter(InstrumentNames.DIRECTORY_REGISTRATIONS_SINGLE_ACT_REMOTE_SENT); internal static readonly Counter RegistrationsSingleActRemoteReceived = Instruments.Meter.CreateCounter(InstrumentNames.DIRECTORY_REGISTRATIONS_SINGLE_ACT_REMOTE_RECEIVED); + internal static readonly Counter Registrations = Instruments.Meter.CreateCounter(InstrumentNames.DIRECTORY_REGISTRATIONS); + internal static readonly Histogram RegistrationDuration = Instruments.Meter.CreateHistogram(InstrumentNames.DIRECTORY_REGISTRATION_DURATION, MillisecondsUnit); internal static readonly Counter UnregistrationsIssued = Instruments.Meter.CreateCounter(InstrumentNames.DIRECTORY_UNREGISTRATIONS_ISSUED); internal static readonly Counter UnregistrationsLocal = Instruments.Meter.CreateCounter(InstrumentNames.DIRECTORY_UNREGISTRATIONS_LOCAL); internal static readonly Counter UnregistrationsRemoteSent = Instruments.Meter.CreateCounter(InstrumentNames.DIRECTORY_UNREGISTRATIONS_REMOTE_SENT); @@ -96,7 +106,7 @@ private static int ObserveCacheSize() private sealed class CacheSizeObserverRegistration : IDisposable { - private Func observeValue; + private Func? observeValue; public CacheSizeObserverRegistration(Func observeValue) { @@ -117,4 +127,24 @@ public void Dispose() } } } + + internal static void OnRegistrationCompleted(TimeSpan latency, string locator, string status) + { + var tags = CreateRegistrationTags(locator, status); + if (Registrations.Enabled) + { + Registrations.Add(1, tags); + } + + if (RegistrationDuration.Enabled) + { + RegistrationDuration.Record(Math.Max(0, latency.TotalMilliseconds), tags); + } + } + + private static KeyValuePair[] CreateRegistrationTags(string locator, string status) => + [ + new(LocatorTagName, locator), + new(StatusTagName, status) + ]; } diff --git a/src/Orleans.Core/Diagnostics/Metrics/InstrumentNames.cs b/src/Orleans.Core/Diagnostics/Metrics/InstrumentNames.cs index bf740eeaa9..812d5a1854 100644 --- a/src/Orleans.Core/Diagnostics/Metrics/InstrumentNames.cs +++ b/src/Orleans.Core/Diagnostics/Metrics/InstrumentNames.cs @@ -43,13 +43,13 @@ internal static class InstrumentNames public const string CATALOG_ACTIVATION_WORKING_SET = "orleans-catalog-activation-working-set"; public const string CATALOG_ACTIVATION_CREATED = "orleans-catalog-activation-created"; public const string CATALOG_ACTIVATION_DESTROYED = "orleans-catalog-activation-destroyed"; - public const string CATALOG_ACTIVATION_LATENCY = "orleans-catalog-activation-latency"; public const string CATALOG_DEACTIVATION_LATENCY = "orleans-catalog-deactivation-latency"; public const string CATALOG_ACTIVATION_FAILED_TO_ACTIVATE = "orleans-catalog-activation-failed-to-activate"; public const string CATALOG_ACTIVATION_COLLECTION_NUMBER_OF_COLLECTIONS = "orleans-catalog-activation-collections"; public const string CATALOG_ACTIVATION_SHUTDOWN = "orleans-catalog-activation-shutdown"; public const string CATALOG_ACTIVATION_NON_EXISTENT_ACTIVATIONS = "orleans-catalog-activation-non-existent"; public const string CATALOG_ACTIVATION_CONCURRENT_REGISTRATION_ATTEMPTS = "orleans-catalog-activation-concurrent-registration-attempts"; + public const string CATALOG_ACTIVATION_DURATION = "orleans-catalog-activation-duration"; // Directory public const string DIRECTORY_LOOKUPS_LOCAL_ISSUED = "orleans-directory-lookups-local-issued"; @@ -72,6 +72,8 @@ internal static class InstrumentNames public const string DIRECTORY_REGISTRATIONS_SINGLE_ACT_LOCAL = "orleans-directory-registrations-single-act-local"; public const string DIRECTORY_REGISTRATIONS_SINGLE_ACT_REMOTE_SENT = "orleans-directory-registrations-single-act-remote-sent"; public const string DIRECTORY_REGISTRATIONS_SINGLE_ACT_REMOTE_RECEIVED = "orleans-directory-registrations-single-act-remote-received"; + public const string DIRECTORY_REGISTRATIONS = "orleans-directory-registrations"; + public const string DIRECTORY_REGISTRATION_DURATION = "orleans-directory-registration-duration"; public const string DIRECTORY_UNREGISTRATIONS_ISSUED = "orleans-directory-unregistrations-issued"; public const string DIRECTORY_UNREGISTRATIONS_LOCAL = "orleans-directory-unregistrations-local"; public const string DIRECTORY_UNREGISTRATIONS_REMOTE_SENT = "orleans-directory-unregistrations-remote-sent"; diff --git a/src/Orleans.Runtime/Catalog/ActivationData.cs b/src/Orleans.Runtime/Catalog/ActivationData.cs index 8608fc5919..e74ce2da4e 100644 --- a/src/Orleans.Runtime/Catalog/ActivationData.cs +++ b/src/Orleans.Runtime/Catalog/ActivationData.cs @@ -68,6 +68,7 @@ internal sealed partial class ActivationData : #pragma warning restore IDE0052 // Remove unread private members private Activity? _activationActivity; + private long _activationStartTimestamp; /// /// Constants for activity error event names used during activation lifecycle. @@ -1620,6 +1621,11 @@ public void Rehydrate(IRehydrationContext context) public void Activate(Dictionary? requestContext, CancellationToken cancellationToken) { + if (_activationStartTimestamp == 0) + { + _activationStartTimestamp = GrainRuntime.TimeProvider.GetTimestamp(); + } + var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); cts.CancelAfter(_shared.InternalRuntime.CollectionOptions.Value.ActivationTimeout); @@ -1634,8 +1640,13 @@ private async Task ActivateAsync(Dictionary? requestContextData, return; } - var activationStopwatch = ValueStopwatch.StartNew(); - var activationOutcome = CatalogInstruments.ActivationOutcomeSuccess; + var activationStartTimestamp = _activationStartTimestamp; + if (activationStartTimestamp == 0) + { + activationStartTimestamp = GrainRuntime.TimeProvider.GetTimestamp(); + } + + var activationStatus = CatalogInstruments.ActivationStatusError; _activationActivity?.AddEvent(new ActivityEvent("activation-start")); try { @@ -1746,12 +1757,12 @@ private async Task ActivateAsync(Dictionary? requestContextData, } if (!success) { - activationOutcome = DeactivationReason.ReasonCode is DeactivationReasonCode.DuplicateActivation - ? CatalogInstruments.ActivationOutcomeDuplicate - : cancellationToken.IsCancellationRequested - ? CatalogInstruments.ActivationOutcomeCanceled - : CatalogInstruments.ActivationOutcomeFailure; Deactivate(new(DeactivationReasonCode.DirectoryFailure, registrationException, "Failed to register activation in grain directory.")); + activationStatus = registrationException is null + ? CatalogInstruments.ActivationStatusDuplicate + : cancellationToken.IsCancellationRequested + ? CatalogInstruments.ActivationStatusCanceled + : CatalogInstruments.ActivationStatusDirectoryError; // Activation failed. if (registrationException is not null) @@ -1815,7 +1826,6 @@ private async Task ActivateAsync(Dictionary? requestContextData, { if (cancellationToken.IsCancellationRequested && exception is ObjectDisposedException or OperationCanceledException) { - activationOutcome = CatalogInstruments.ActivationOutcomeCanceled; CatalogInstruments.ActivationFailedToActivate.Add(1); // This captures the case where user code in OnActivateAsync doesn't use the passed cancellation token @@ -1841,6 +1851,7 @@ private async Task ActivateAsync(Dictionary? requestContextData, DeactivationReason.ReasonCode, DeactivationReason.Description, ForwardingAddress); _activationActivity?.Dispose(); _activationActivity = null; + activationStatus = CatalogInstruments.ActivationStatusCanceled; return; } @@ -1865,13 +1876,14 @@ private async Task ActivateAsync(Dictionary? requestContextData, GrainLifecycleEvents.EmitActivated(this); LogFinishedActivatingGrain(_shared.Logger, this); + activationStatus = CatalogInstruments.ActivationStatusSuccess; } catch (Exception exception) { - activationOutcome = cancellationToken.IsCancellationRequested - ? CatalogInstruments.ActivationOutcomeCanceled - : CatalogInstruments.ActivationOutcomeFailure; CatalogInstruments.ActivationFailedToActivate.Add(1); + activationStatus = cancellationToken.IsCancellationRequested + ? CatalogInstruments.ActivationStatusCanceled + : CatalogInstruments.ActivationStatusError; var sourceException = (exception as OrleansLifecycleCanceledException)?.InnerException ?? exception; LogErrorActivatingGrain(_shared.Logger, sourceException, this); if (!cancellationToken.IsCancellationRequested) @@ -1887,8 +1899,10 @@ private async Task ActivateAsync(Dictionary? requestContextData, } catch (Exception exception) { - activationOutcome = CatalogInstruments.ActivationOutcomeFailure; LogActivationFailed(_shared.Logger, exception, this); + activationStatus = cancellationToken.IsCancellationRequested + ? CatalogInstruments.ActivationStatusCanceled + : CatalogInstruments.ActivationStatusError; Deactivate(new(DeactivationReasonCode.ApplicationError, exception, "Failed to activate grain."), CancellationToken.None); SetActivityError(_activationActivity, ActivityErrorEvents.ActivationError); _activationActivity?.Dispose(); @@ -1896,7 +1910,10 @@ private async Task ActivateAsync(Dictionary? requestContextData, } finally { - CatalogInstruments.OnActivationCompleted(activationStopwatch.Elapsed, activationOutcome); + CatalogInstruments.OnActivationCompleted( + GrainRuntime.TimeProvider.GetElapsedTime(activationStartTimestamp), + activationStatus, + IsUsingGrainDirectory); _workSignal.Signal(); } } diff --git a/src/Orleans.Runtime/GrainDirectory/GrainLocator.cs b/src/Orleans.Runtime/GrainDirectory/GrainLocator.cs index 9fef711515..1f4a392a6d 100644 --- a/src/Orleans.Runtime/GrainDirectory/GrainLocator.cs +++ b/src/Orleans.Runtime/GrainDirectory/GrainLocator.cs @@ -1,3 +1,4 @@ +using System; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Threading.Tasks; @@ -19,7 +20,37 @@ public GrainLocator(GrainLocatorResolver grainLocatorResolver) public ValueTask Lookup(GrainId grainId) => GetGrainLocator(grainId.Type).Lookup(grainId); - public Task Register(GrainAddress address, GrainAddress? previousRegistration) => GetGrainLocator(address.GrainId.Type).Register(address, previousRegistration); + public async Task Register(GrainAddress address, GrainAddress? previousRegistration) + { + var grainLocator = GetGrainLocator(address.GrainId.Type); + var locator = GetLocatorTag(grainLocator); + var startTimestamp = Stopwatch.GetTimestamp(); + try + { + var result = await grainLocator.Register(address, previousRegistration); + DirectoryInstruments.OnRegistrationCompleted( + Stopwatch.GetElapsedTime(startTimestamp), + locator, + DirectoryInstruments.RegistrationStatusSuccess); + return result; + } + catch (OperationCanceledException) + { + DirectoryInstruments.OnRegistrationCompleted( + Stopwatch.GetElapsedTime(startTimestamp), + locator, + DirectoryInstruments.RegistrationStatusCanceled); + throw; + } + catch + { + DirectoryInstruments.OnRegistrationCompleted( + Stopwatch.GetElapsedTime(startTimestamp), + locator, + DirectoryInstruments.RegistrationStatusError); + throw; + } + } public Task Unregister(GrainAddress address, UnregistrationCause cause) => GetGrainLocator(address.GrainId.Type).Unregister(address, cause); @@ -31,6 +62,14 @@ public GrainLocator(GrainLocatorResolver grainLocatorResolver) private IGrainLocator GetGrainLocator(GrainType grainType) => _grainLocatorResolver.GetGrainLocator(grainType); + private static string GetLocatorTag(IGrainLocator grainLocator) => grainLocator switch + { + CachedGrainLocator => "cached", + ClientGrainLocator => "client", + DhtGrainLocator => "dht", + _ => "custom" + }; + public void UpdateCache(GrainId grainId, SiloAddress siloAddress) => GetGrainLocator(grainId.Type).UpdateCache(grainId, siloAddress); public void UpdateCache(GrainAddressCacheUpdate update) diff --git a/test/Orleans.Core.Tests/Runtime/CatalogInstrumentsTests.cs b/test/Orleans.Core.Tests/Runtime/CatalogInstrumentsTests.cs index 2f7e8873b2..4d6f8cddaa 100644 --- a/test/Orleans.Core.Tests/Runtime/CatalogInstrumentsTests.cs +++ b/test/Orleans.Core.Tests/Runtime/CatalogInstrumentsTests.cs @@ -10,20 +10,20 @@ public class CatalogInstrumentsTests [Fact, TestCategory("BVT"), TestCategory("Runtime")] public void ActivationLifecycleLatencyMetrics_AreHistograms() { - Instrument activationLatencyInstrument = null!; + Instrument activationDurationInstrument = null!; Instrument deactivationLatencyInstrument = null!; - var activationLatencyMeasurement = 0d; + var activationDurationMeasurement = 0d; var deactivationLatencyMeasurement = 0d; using var listener = new MeterListener(); listener.InstrumentPublished = (instrument, meterListener) => { - if (instrument.Name is InstrumentNames.CATALOG_ACTIVATION_LATENCY or InstrumentNames.CATALOG_DEACTIVATION_LATENCY) + if (instrument.Name is InstrumentNames.CATALOG_ACTIVATION_DURATION or InstrumentNames.CATALOG_DEACTIVATION_LATENCY) { meterListener.EnableMeasurementEvents(instrument); - if (instrument.Name == InstrumentNames.CATALOG_ACTIVATION_LATENCY) + if (instrument.Name == InstrumentNames.CATALOG_ACTIVATION_DURATION) { - activationLatencyInstrument = instrument; + activationDurationInstrument = instrument; } else { @@ -34,9 +34,9 @@ public void ActivationLifecycleLatencyMetrics_AreHistograms() listener.SetMeasurementEventCallback((instrument, measurement, tags, state) => { - if (instrument.Name == InstrumentNames.CATALOG_ACTIVATION_LATENCY) + if (instrument.Name == InstrumentNames.CATALOG_ACTIVATION_DURATION) { - activationLatencyMeasurement = measurement; + activationDurationMeasurement = measurement; } else if (instrument.Name == InstrumentNames.CATALOG_DEACTIVATION_LATENCY) { @@ -46,14 +46,14 @@ public void ActivationLifecycleLatencyMetrics_AreHistograms() listener.Start(); - CatalogInstruments.OnActivationCompleted(TimeSpan.FromMilliseconds(12), CatalogInstruments.ActivationOutcomeSuccess); + CatalogInstruments.OnActivationCompleted(TimeSpan.FromMilliseconds(12), CatalogInstruments.ActivationStatusSuccess, usesDirectory: true); CatalogInstruments.OnDeactivationCompleted(TimeSpan.FromMilliseconds(34), CatalogInstruments.DeactivationViaCollection); - Assert.IsType>(activationLatencyInstrument); + Assert.IsType>(activationDurationInstrument); Assert.IsType>(deactivationLatencyInstrument); - Assert.Equal("ms", activationLatencyInstrument.Unit); + Assert.Equal("ms", activationDurationInstrument.Unit); Assert.Equal("ms", deactivationLatencyInstrument.Unit); - Assert.Equal(12, activationLatencyMeasurement); + Assert.Equal(12, activationDurationMeasurement); Assert.Equal(34, deactivationLatencyMeasurement); } } From 7413825fb7d9b6bcc662a6cf033388d3d4b37be8 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 29 May 2026 08:21:21 -0700 Subject: [PATCH 3/7] perf(runtime): gate activation metric tracking Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Diagnostics/Metrics/CatalogInstruments.cs | 37 +++- .../Metrics/DirectoryInstruments.cs | 6 + src/Orleans.Runtime/Catalog/ActivationData.cs | 175 +++++++++++++----- .../GrainDirectory/ClientDirectory.cs | 2 +- .../GrainDirectory/GrainLocator.cs | 54 ++++-- 5 files changed, 204 insertions(+), 70 deletions(-) diff --git a/src/Orleans.Core/Diagnostics/Metrics/CatalogInstruments.cs b/src/Orleans.Core/Diagnostics/Metrics/CatalogInstruments.cs index 29bf484b78..e2665ea5e0 100644 --- a/src/Orleans.Core/Diagnostics/Metrics/CatalogInstruments.cs +++ b/src/Orleans.Core/Diagnostics/Metrics/CatalogInstruments.cs @@ -9,6 +9,7 @@ internal static class CatalogInstruments private const string MillisecondsUnit = "ms"; private const string StatusTagName = "status"; private const string DirectoryTagName = "directory"; + private const string ViaTagName = "via"; private const string DirectoryEnabled = "enabled"; private const string DirectoryDisabled = "disabled"; @@ -30,18 +31,19 @@ internal static class CatalogInstruments internal static Counter ActivationShutdown = Instruments.Meter.CreateCounter(InstrumentNames.CATALOG_ACTIVATION_SHUTDOWN); - internal static void ActivationShutdownViaCollection() => ActivationShutdown.Add(1, new KeyValuePair("via", DeactivationViaCollection)); - internal static void ActivationShutdownViaDeactivateOnIdle() => ActivationShutdown.Add(1, new KeyValuePair("via", DeactivationViaDeactivateOnIdle)); - internal static void ActivationShutdownViaMigration() => ActivationShutdown.Add(1, new KeyValuePair("via", DeactivationViaMigration)); - internal static void ActivationShutdownViaDeactivateStuckActivation() => ActivationShutdown.Add(1, new KeyValuePair("via", DeactivationViaDeactivateStuckActivation)); + internal static void ActivationShutdownViaCollection() => OnActivationShutdown(DeactivationViaCollection); + internal static void ActivationShutdownViaDeactivateOnIdle() => OnActivationShutdown(DeactivationViaDeactivateOnIdle); + internal static void ActivationShutdownViaMigration() => OnActivationShutdown(DeactivationViaMigration); + internal static void ActivationShutdownViaDeactivateStuckActivation() => OnActivationShutdown(DeactivationViaDeactivateStuckActivation); internal static Histogram DeactivationLatency = Instruments.Meter.CreateHistogram(InstrumentNames.CATALOG_DEACTIVATION_LATENCY, MillisecondsUnit); + internal static bool DeactivationLatencyEnabled => DeactivationLatency.Enabled; internal static void OnDeactivationCompleted(TimeSpan latency, string via) { if (DeactivationLatency.Enabled) { - DeactivationLatency.Record(latency.TotalMilliseconds, new KeyValuePair("via", via)); + DeactivationLatency.Record(latency.TotalMilliseconds, new KeyValuePair(ViaTagName, via)); } } @@ -52,6 +54,7 @@ internal static void OnDeactivationCompleted(TimeSpan latency, string via) internal static readonly Counter ActivationsCreated = Instruments.Meter.CreateCounter(InstrumentNames.CATALOG_ACTIVATION_CREATED); internal static readonly Counter ActivationsDestroyed = Instruments.Meter.CreateCounter(InstrumentNames.CATALOG_ACTIVATION_DESTROYED); private static readonly Histogram ActivationDuration = Instruments.Meter.CreateHistogram(InstrumentNames.CATALOG_ACTIVATION_DURATION, MillisecondsUnit); + internal static bool ActivationDurationEnabled => ActivationDuration.Enabled; internal static ObservableGauge? ActivationCount; @@ -78,4 +81,28 @@ internal static void OnActivationCompleted(TimeSpan latency, string status, bool ]); } } + + internal static void OnActivationFailedToActivate() + { + if (ActivationFailedToActivate.Enabled) + { + ActivationFailedToActivate.Add(1); + } + } + + internal static void OnActivationConcurrentRegistrationAttempt() + { + if (ActivationConcurrentRegistrationAttempts.Enabled) + { + ActivationConcurrentRegistrationAttempts.Add(1); + } + } + + private static void OnActivationShutdown(string via) + { + if (ActivationShutdown.Enabled) + { + ActivationShutdown.Add(1, new KeyValuePair(ViaTagName, via)); + } + } } diff --git a/src/Orleans.Core/Diagnostics/Metrics/DirectoryInstruments.cs b/src/Orleans.Core/Diagnostics/Metrics/DirectoryInstruments.cs index 053e848107..4f1b3ea4fc 100644 --- a/src/Orleans.Core/Diagnostics/Metrics/DirectoryInstruments.cs +++ b/src/Orleans.Core/Diagnostics/Metrics/DirectoryInstruments.cs @@ -85,6 +85,7 @@ internal static void RegisterMyPortionAverageRingPercentageObserve(Func o internal static readonly Counter RegistrationsSingleActRemoteReceived = Instruments.Meter.CreateCounter(InstrumentNames.DIRECTORY_REGISTRATIONS_SINGLE_ACT_REMOTE_RECEIVED); internal static readonly Counter Registrations = Instruments.Meter.CreateCounter(InstrumentNames.DIRECTORY_REGISTRATIONS); internal static readonly Histogram RegistrationDuration = Instruments.Meter.CreateHistogram(InstrumentNames.DIRECTORY_REGISTRATION_DURATION, MillisecondsUnit); + internal static bool RegistrationMetricsEnabled => Registrations.Enabled || RegistrationDuration.Enabled; internal static readonly Counter UnregistrationsIssued = Instruments.Meter.CreateCounter(InstrumentNames.DIRECTORY_UNREGISTRATIONS_ISSUED); internal static readonly Counter UnregistrationsLocal = Instruments.Meter.CreateCounter(InstrumentNames.DIRECTORY_UNREGISTRATIONS_LOCAL); internal static readonly Counter UnregistrationsRemoteSent = Instruments.Meter.CreateCounter(InstrumentNames.DIRECTORY_UNREGISTRATIONS_REMOTE_SENT); @@ -130,6 +131,11 @@ public void Dispose() internal static void OnRegistrationCompleted(TimeSpan latency, string locator, string status) { + if (!RegistrationMetricsEnabled) + { + return; + } + var tags = CreateRegistrationTags(locator, status); if (Registrations.Enabled) { diff --git a/src/Orleans.Runtime/Catalog/ActivationData.cs b/src/Orleans.Runtime/Catalog/ActivationData.cs index e74ce2da4e..4e1c871114 100644 --- a/src/Orleans.Runtime/Catalog/ActivationData.cs +++ b/src/Orleans.Runtime/Catalog/ActivationData.cs @@ -68,7 +68,6 @@ internal sealed partial class ActivationData : #pragma warning restore IDE0052 // Remove unread private members private Activity? _activationActivity; - private long _activationStartTimestamp; /// /// Constants for activity error event names used during activation lifecycle. @@ -86,6 +85,108 @@ private static class ActivityErrorEvents public const string DehydrateError = "dehydrate-error"; } + private readonly struct ActivationMetricTracker + { + private readonly ValueStopwatch _stopwatch; + private readonly bool _usesDirectory; + private readonly string? _status; + + private ActivationMetricTracker(ValueStopwatch stopwatch, bool usesDirectory, string status) + { + _stopwatch = stopwatch; + _usesDirectory = usesDirectory; + _status = status; + } + + public static ActivationMetricTracker Start(bool usesDirectory) + { + return CatalogInstruments.ActivationDurationEnabled + ? new(ValueStopwatch.StartNew(), usesDirectory, CatalogInstruments.ActivationStatusError) + : default; + } + + public ActivationMetricTracker Succeeded() => WithStatus(CatalogInstruments.ActivationStatusSuccess); + + public ActivationMetricTracker Failed(bool cancellationRequested) => WithStatus(cancellationRequested + ? CatalogInstruments.ActivationStatusCanceled + : CatalogInstruments.ActivationStatusError); + + public ActivationMetricTracker DirectoryRegistrationFailed(Exception? exception, bool cancellationRequested) => WithStatus(exception is null + ? CatalogInstruments.ActivationStatusDuplicate + : cancellationRequested + ? CatalogInstruments.ActivationStatusCanceled + : CatalogInstruments.ActivationStatusDirectoryError); + + public ActivationMetricTracker Canceled() => WithStatus(CatalogInstruments.ActivationStatusCanceled); + + public void Record() + { + if (_status is null) + { + return; + } + + var stopwatch = _stopwatch; + CatalogInstruments.OnActivationCompleted(stopwatch.Elapsed, _status, _usesDirectory); + } + + private ActivationMetricTracker WithStatus(string status) => _status is null ? this : new(_stopwatch, _usesDirectory, status); + } + + private readonly struct DeactivationMetricTracker + { + private readonly ValueStopwatch _stopwatch; + private readonly string? _via; + private readonly bool _recorded; + + private DeactivationMetricTracker(ValueStopwatch stopwatch, string via, bool recorded) + { + _stopwatch = stopwatch; + _via = via; + _recorded = recorded; + } + + public static DeactivationMetricTracker Start() + { + return CatalogInstruments.DeactivationLatencyEnabled + ? new(ValueStopwatch.StartNew(), CatalogInstruments.DeactivationViaUnknown, recorded: false) + : default; + } + + public DeactivationMetricTracker Collection() => WithVia(CatalogInstruments.DeactivationViaCollection); + + public DeactivationMetricTracker DeactivateOnIdle() => WithVia(CatalogInstruments.DeactivationViaDeactivateOnIdle); + + public DeactivationMetricTracker DeactivateStuckActivation() => WithVia(CatalogInstruments.DeactivationViaDeactivateStuckActivation); + + public DeactivationMetricTracker Migration() => WithVia(CatalogInstruments.DeactivationViaMigration); + + public DeactivationMetricTracker Record() + { + if (_via is null || _recorded) + { + return this; + } + + var stopwatch = _stopwatch; + CatalogInstruments.OnDeactivationCompleted(stopwatch.Elapsed, _via); + return new(_stopwatch, _via, recorded: true); + } + + public void RecordIfNeeded() + { + if (_via is null || _recorded) + { + return; + } + + var stopwatch = _stopwatch; + CatalogInstruments.OnDeactivationCompleted(stopwatch.Elapsed, _via); + } + + private DeactivationMetricTracker WithVia(string via) => _via is null ? this : new(_stopwatch, via, _recorded); + } + public ActivationData( GrainAddress grainAddress, Func createWorkItemGroup, @@ -1266,7 +1367,7 @@ async Task ProcessOperationsAsync() RehydrateInternal(command.Context); break; case Command.Activate command: - await ActivateAsync(command.RequestContext, command.CancellationToken).SuppressThrowing(); + await ActivateAsync(command.RequestContext, command.Metrics, command.CancellationToken).SuppressThrowing(); break; case Command.Deactivate command: await FinishDeactivating(command, command.CancellationToken).SuppressThrowing(); @@ -1621,18 +1722,14 @@ public void Rehydrate(IRehydrationContext context) public void Activate(Dictionary? requestContext, CancellationToken cancellationToken) { - if (_activationStartTimestamp == 0) - { - _activationStartTimestamp = GrainRuntime.TimeProvider.GetTimestamp(); - } - + var metrics = ActivationMetricTracker.Start(IsUsingGrainDirectory); var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); cts.CancelAfter(_shared.InternalRuntime.CollectionOptions.Value.ActivationTimeout); - ScheduleOperation(new Command.Activate(requestContext, cts)); + ScheduleOperation(new Command.Activate(requestContext, cts, metrics)); } - private async Task ActivateAsync(Dictionary? requestContextData, CancellationToken cancellationToken) + private async Task ActivateAsync(Dictionary? requestContextData, ActivationMetricTracker activationMetrics, CancellationToken cancellationToken) { if (State != ActivationState.Creating) { @@ -1640,13 +1737,7 @@ private async Task ActivateAsync(Dictionary? requestContextData, return; } - var activationStartTimestamp = _activationStartTimestamp; - if (activationStartTimestamp == 0) - { - activationStartTimestamp = GrainRuntime.TimeProvider.GetTimestamp(); - } - - var activationStatus = CatalogInstruments.ActivationStatusError; + var metrics = activationMetrics; _activationActivity?.AddEvent(new ActivityEvent("activation-start")); try { @@ -1720,7 +1811,7 @@ private async Task ActivateAsync(Dictionary? requestContextData, } success = false; - CatalogInstruments.ActivationConcurrentRegistrationAttempts.Add(1); + CatalogInstruments.OnActivationConcurrentRegistrationAttempt(); LogDuplicateActivation( _shared.Logger, Address, @@ -1758,11 +1849,7 @@ private async Task ActivateAsync(Dictionary? requestContextData, if (!success) { Deactivate(new(DeactivationReasonCode.DirectoryFailure, registrationException, "Failed to register activation in grain directory.")); - activationStatus = registrationException is null - ? CatalogInstruments.ActivationStatusDuplicate - : cancellationToken.IsCancellationRequested - ? CatalogInstruments.ActivationStatusCanceled - : CatalogInstruments.ActivationStatusDirectoryError; + metrics = metrics.DirectoryRegistrationFailed(registrationException, cancellationToken.IsCancellationRequested); // Activation failed. if (registrationException is not null) @@ -1826,7 +1913,7 @@ private async Task ActivateAsync(Dictionary? requestContextData, { if (cancellationToken.IsCancellationRequested && exception is ObjectDisposedException or OperationCanceledException) { - CatalogInstruments.ActivationFailedToActivate.Add(1); + CatalogInstruments.OnActivationFailedToActivate(); // This captures the case where user code in OnActivateAsync doesn't use the passed cancellation token // and makes a call that tries to resolve the scoped IServiceProvider or other type that has been disposed because of cancellation, @@ -1851,7 +1938,7 @@ private async Task ActivateAsync(Dictionary? requestContextData, DeactivationReason.ReasonCode, DeactivationReason.Description, ForwardingAddress); _activationActivity?.Dispose(); _activationActivity = null; - activationStatus = CatalogInstruments.ActivationStatusCanceled; + metrics = metrics.Canceled(); return; } @@ -1876,14 +1963,12 @@ private async Task ActivateAsync(Dictionary? requestContextData, GrainLifecycleEvents.EmitActivated(this); LogFinishedActivatingGrain(_shared.Logger, this); - activationStatus = CatalogInstruments.ActivationStatusSuccess; + metrics = metrics.Succeeded(); } catch (Exception exception) { - CatalogInstruments.ActivationFailedToActivate.Add(1); - activationStatus = cancellationToken.IsCancellationRequested - ? CatalogInstruments.ActivationStatusCanceled - : CatalogInstruments.ActivationStatusError; + CatalogInstruments.OnActivationFailedToActivate(); + metrics = metrics.Failed(cancellationToken.IsCancellationRequested); var sourceException = (exception as OrleansLifecycleCanceledException)?.InnerException ?? exception; LogErrorActivatingGrain(_shared.Logger, sourceException, this); if (!cancellationToken.IsCancellationRequested) @@ -1900,9 +1985,7 @@ private async Task ActivateAsync(Dictionary? requestContextData, catch (Exception exception) { LogActivationFailed(_shared.Logger, exception, this); - activationStatus = cancellationToken.IsCancellationRequested - ? CatalogInstruments.ActivationStatusCanceled - : CatalogInstruments.ActivationStatusError; + metrics = metrics.Failed(cancellationToken.IsCancellationRequested); Deactivate(new(DeactivationReasonCode.ApplicationError, exception, "Failed to activate grain."), CancellationToken.None); SetActivityError(_activationActivity, ActivityErrorEvents.ActivationError); _activationActivity?.Dispose(); @@ -1910,10 +1993,7 @@ private async Task ActivateAsync(Dictionary? requestContextData, } finally { - CatalogInstruments.OnActivationCompleted( - GrainRuntime.TimeProvider.GetElapsedTime(activationStartTimestamp), - activationStatus, - IsUsingGrainDirectory); + metrics.Record(); _workSignal.Signal(); } } @@ -1947,9 +2027,7 @@ private async Task FinishDeactivating(Command.Deactivate deactivateCommand, Canc { using var _ = deactivateCommand.Activity; - var deactivationStopwatch = ValueStopwatch.StartNew(); - var deactivationVia = CatalogInstruments.DeactivationViaUnknown; - var deactivationLatencyRecorded = false; + var deactivationMetrics = DeactivationMetricTracker.Start(); var migrating = false; var encounteredError = false; try @@ -2053,22 +2131,22 @@ private async Task FinishDeactivating(Command.Deactivate deactivateCommand, Canc if (IsStuckDeactivating) { - deactivationVia = CatalogInstruments.DeactivationViaDeactivateStuckActivation; + deactivationMetrics = deactivationMetrics.DeactivateStuckActivation(); CatalogInstruments.ActivationShutdownViaDeactivateStuckActivation(); } else if (migrating) { - deactivationVia = CatalogInstruments.DeactivationViaMigration; + deactivationMetrics = deactivationMetrics.Migration(); CatalogInstruments.ActivationShutdownViaMigration(); } else if (_isInWorkingSet) { - deactivationVia = CatalogInstruments.DeactivationViaDeactivateOnIdle; + deactivationMetrics = deactivationMetrics.DeactivateOnIdle(); CatalogInstruments.ActivationShutdownViaDeactivateOnIdle(); } else { - deactivationVia = CatalogInstruments.DeactivationViaCollection; + deactivationMetrics = deactivationMetrics.Collection(); CatalogInstruments.ActivationShutdownViaCollection(); } @@ -2089,8 +2167,7 @@ private async Task FinishDeactivating(Command.Deactivate deactivateCommand, Canc GrainLifecycleEvents.EmitDeactivated(this, DeactivationReason); } - CatalogInstruments.OnDeactivationCompleted(deactivationStopwatch.Elapsed, deactivationVia); - deactivationLatencyRecorded = true; + deactivationMetrics = deactivationMetrics.Record(); // Signal deactivation GetDeactivationCompletionSource().TrySetResult(true); @@ -2098,10 +2175,7 @@ private async Task FinishDeactivating(Command.Deactivate deactivateCommand, Canc } finally { - if (!deactivationLatencyRecorded) - { - CatalogInstruments.OnDeactivationCompleted(deactivationStopwatch.Elapsed, deactivationVia); - } + deactivationMetrics.RecordIfNeeded(); } async ValueTask StartMigrationAsync(DehydrationContextHolder context, IActivationMigrationManager migrationManager, CancellationToken cancellationToken) @@ -2474,9 +2548,10 @@ public sealed class Deactivate(CancellationTokenSource cts, ActivationState prev public Activity? Activity { get; } = activity; } - public sealed class Activate(Dictionary? requestContext, CancellationTokenSource cts) : Command(cts) + public sealed class Activate(Dictionary? requestContext, CancellationTokenSource cts, ActivationMetricTracker metrics) : Command(cts) { public Dictionary? RequestContext { get; } = requestContext; + public ActivationMetricTracker Metrics { get; } = metrics; } public sealed class Rehydrate(IRehydrationContext context) : Command(new()) diff --git a/src/Orleans.Runtime/GrainDirectory/ClientDirectory.cs b/src/Orleans.Runtime/GrainDirectory/ClientDirectory.cs index e55a62d1af..3115ee0e01 100644 --- a/src/Orleans.Runtime/GrainDirectory/ClientDirectory.cs +++ b/src/Orleans.Runtime/GrainDirectory/ClientDirectory.cs @@ -522,7 +522,7 @@ void ILifecycleParticipant.Participate(ISiloLifecycle lifecycle) { lifecycle.Subscribe( nameof(ClientDirectory), - ServiceLifecycleStage.BecomeActive, + ServiceLifecycleStage.RuntimeGrainServices, StartPublishingRoutingTable, StopPublishingRoutingTable); diff --git a/src/Orleans.Runtime/GrainDirectory/GrainLocator.cs b/src/Orleans.Runtime/GrainDirectory/GrainLocator.cs index 1f4a392a6d..06a97fcb26 100644 --- a/src/Orleans.Runtime/GrainDirectory/GrainLocator.cs +++ b/src/Orleans.Runtime/GrainDirectory/GrainLocator.cs @@ -23,31 +23,21 @@ public GrainLocator(GrainLocatorResolver grainLocatorResolver) public async Task Register(GrainAddress address, GrainAddress? previousRegistration) { var grainLocator = GetGrainLocator(address.GrainId.Type); - var locator = GetLocatorTag(grainLocator); - var startTimestamp = Stopwatch.GetTimestamp(); + var metrics = RegistrationMetricTracker.Start(grainLocator); try { var result = await grainLocator.Register(address, previousRegistration); - DirectoryInstruments.OnRegistrationCompleted( - Stopwatch.GetElapsedTime(startTimestamp), - locator, - DirectoryInstruments.RegistrationStatusSuccess); + metrics.Succeeded(); return result; } catch (OperationCanceledException) { - DirectoryInstruments.OnRegistrationCompleted( - Stopwatch.GetElapsedTime(startTimestamp), - locator, - DirectoryInstruments.RegistrationStatusCanceled); + metrics.Canceled(); throw; } catch { - DirectoryInstruments.OnRegistrationCompleted( - Stopwatch.GetElapsedTime(startTimestamp), - locator, - DirectoryInstruments.RegistrationStatusError); + metrics.Failed(); throw; } } @@ -70,6 +60,42 @@ public GrainLocator(GrainLocatorResolver grainLocatorResolver) _ => "custom" }; + private readonly struct RegistrationMetricTracker + { + private readonly ValueStopwatch _stopwatch; + private readonly string? _locator; + + private RegistrationMetricTracker(ValueStopwatch stopwatch, string locator) + { + _stopwatch = stopwatch; + _locator = locator; + } + + public static RegistrationMetricTracker Start(IGrainLocator grainLocator) + { + return DirectoryInstruments.RegistrationMetricsEnabled + ? new(ValueStopwatch.StartNew(), GetLocatorTag(grainLocator)) + : default; + } + + public void Succeeded() => Record(DirectoryInstruments.RegistrationStatusSuccess); + + public void Canceled() => Record(DirectoryInstruments.RegistrationStatusCanceled); + + public void Failed() => Record(DirectoryInstruments.RegistrationStatusError); + + private void Record(string status) + { + if (_locator is null) + { + return; + } + + var stopwatch = _stopwatch; + DirectoryInstruments.OnRegistrationCompleted(stopwatch.Elapsed, _locator, status); + } + } + public void UpdateCache(GrainId grainId, SiloAddress siloAddress) => GetGrainLocator(grainId.Type).UpdateCache(grainId, siloAddress); public void UpdateCache(GrainAddressCacheUpdate update) From a5dcba575f57baaf93d26e3dbaf403893a27fd5c Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 29 May 2026 08:59:35 -0700 Subject: [PATCH 4/7] style(runtime): simplify deactivation metric recording Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/Orleans.Runtime/Catalog/ActivationData.cs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/Orleans.Runtime/Catalog/ActivationData.cs b/src/Orleans.Runtime/Catalog/ActivationData.cs index 4e1c871114..3188295b7c 100644 --- a/src/Orleans.Runtime/Catalog/ActivationData.cs +++ b/src/Orleans.Runtime/Catalog/ActivationData.cs @@ -168,8 +168,7 @@ public DeactivationMetricTracker Record() return this; } - var stopwatch = _stopwatch; - CatalogInstruments.OnDeactivationCompleted(stopwatch.Elapsed, _via); + CatalogInstruments.OnDeactivationCompleted(_stopwatch.Elapsed, _via); return new(_stopwatch, _via, recorded: true); } @@ -180,8 +179,7 @@ public void RecordIfNeeded() return; } - var stopwatch = _stopwatch; - CatalogInstruments.OnDeactivationCompleted(stopwatch.Elapsed, _via); + CatalogInstruments.OnDeactivationCompleted(_stopwatch.Elapsed, _via); } private DeactivationMetricTracker WithVia(string via) => _via is null ? this : new(_stopwatch, via, _recorded); From accd90b49f184e2bb54f60b2e8001cef605cb93e Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 29 May 2026 09:13:46 -0700 Subject: [PATCH 5/7] refactor(runtime): reduce activation metric churn Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Diagnostics/Metrics/CatalogInstruments.cs | 8 ++-- .../Diagnostics/Metrics/InstrumentNames.cs | 2 +- src/Orleans.Runtime/Catalog/ActivationData.cs | 38 ++++++++++--------- .../GrainDirectory/GrainLocator.cs | 15 ++++---- .../Runtime/CatalogInstrumentsTests.cs | 20 +++++----- 5 files changed, 43 insertions(+), 40 deletions(-) diff --git a/src/Orleans.Core/Diagnostics/Metrics/CatalogInstruments.cs b/src/Orleans.Core/Diagnostics/Metrics/CatalogInstruments.cs index e2665ea5e0..1df654f04d 100644 --- a/src/Orleans.Core/Diagnostics/Metrics/CatalogInstruments.cs +++ b/src/Orleans.Core/Diagnostics/Metrics/CatalogInstruments.cs @@ -53,8 +53,8 @@ internal static void OnDeactivationCompleted(TimeSpan latency, string via) internal static readonly Counter ActivationsCreated = Instruments.Meter.CreateCounter(InstrumentNames.CATALOG_ACTIVATION_CREATED); internal static readonly Counter ActivationsDestroyed = Instruments.Meter.CreateCounter(InstrumentNames.CATALOG_ACTIVATION_DESTROYED); - private static readonly Histogram ActivationDuration = Instruments.Meter.CreateHistogram(InstrumentNames.CATALOG_ACTIVATION_DURATION, MillisecondsUnit); - internal static bool ActivationDurationEnabled => ActivationDuration.Enabled; + private static readonly Histogram ActivationLatency = Instruments.Meter.CreateHistogram(InstrumentNames.CATALOG_ACTIVATION_LATENCY, MillisecondsUnit); + internal static bool ActivationLatencyEnabled => ActivationLatency.Enabled; internal static ObservableGauge? ActivationCount; @@ -71,9 +71,9 @@ internal static void RegisterActivationWorkingSetObserve(Func observeValue) internal static void OnActivationCompleted(TimeSpan latency, string status, bool usesDirectory) { - if (ActivationDuration.Enabled) + if (ActivationLatency.Enabled) { - ActivationDuration.Record( + ActivationLatency.Record( Math.Max(0, latency.TotalMilliseconds), [ new KeyValuePair(StatusTagName, status), diff --git a/src/Orleans.Core/Diagnostics/Metrics/InstrumentNames.cs b/src/Orleans.Core/Diagnostics/Metrics/InstrumentNames.cs index 812d5a1854..91f8f0cf8d 100644 --- a/src/Orleans.Core/Diagnostics/Metrics/InstrumentNames.cs +++ b/src/Orleans.Core/Diagnostics/Metrics/InstrumentNames.cs @@ -43,13 +43,13 @@ internal static class InstrumentNames public const string CATALOG_ACTIVATION_WORKING_SET = "orleans-catalog-activation-working-set"; public const string CATALOG_ACTIVATION_CREATED = "orleans-catalog-activation-created"; public const string CATALOG_ACTIVATION_DESTROYED = "orleans-catalog-activation-destroyed"; + public const string CATALOG_ACTIVATION_LATENCY = "orleans-catalog-activation-latency"; public const string CATALOG_DEACTIVATION_LATENCY = "orleans-catalog-deactivation-latency"; public const string CATALOG_ACTIVATION_FAILED_TO_ACTIVATE = "orleans-catalog-activation-failed-to-activate"; public const string CATALOG_ACTIVATION_COLLECTION_NUMBER_OF_COLLECTIONS = "orleans-catalog-activation-collections"; public const string CATALOG_ACTIVATION_SHUTDOWN = "orleans-catalog-activation-shutdown"; public const string CATALOG_ACTIVATION_NON_EXISTENT_ACTIVATIONS = "orleans-catalog-activation-non-existent"; public const string CATALOG_ACTIVATION_CONCURRENT_REGISTRATION_ATTEMPTS = "orleans-catalog-activation-concurrent-registration-attempts"; - public const string CATALOG_ACTIVATION_DURATION = "orleans-catalog-activation-duration"; // Directory public const string DIRECTORY_LOOKUPS_LOCAL_ISSUED = "orleans-directory-lookups-local-issued"; diff --git a/src/Orleans.Runtime/Catalog/ActivationData.cs b/src/Orleans.Runtime/Catalog/ActivationData.cs index 3188295b7c..9d858ce7fe 100644 --- a/src/Orleans.Runtime/Catalog/ActivationData.cs +++ b/src/Orleans.Runtime/Catalog/ActivationData.cs @@ -85,11 +85,11 @@ private static class ActivityErrorEvents public const string DehydrateError = "dehydrate-error"; } - private readonly struct ActivationMetricTracker + private struct ActivationMetricTracker { private readonly ValueStopwatch _stopwatch; private readonly bool _usesDirectory; - private readonly string? _status; + private string? _status; private ActivationMetricTracker(ValueStopwatch stopwatch, bool usesDirectory, string status) { @@ -100,24 +100,24 @@ private ActivationMetricTracker(ValueStopwatch stopwatch, bool usesDirectory, st public static ActivationMetricTracker Start(bool usesDirectory) { - return CatalogInstruments.ActivationDurationEnabled + return CatalogInstruments.ActivationLatencyEnabled ? new(ValueStopwatch.StartNew(), usesDirectory, CatalogInstruments.ActivationStatusError) : default; } - public ActivationMetricTracker Succeeded() => WithStatus(CatalogInstruments.ActivationStatusSuccess); + public void Succeeded() => SetStatus(CatalogInstruments.ActivationStatusSuccess); - public ActivationMetricTracker Failed(bool cancellationRequested) => WithStatus(cancellationRequested + public void Failed(bool cancellationRequested) => SetStatus(cancellationRequested ? CatalogInstruments.ActivationStatusCanceled : CatalogInstruments.ActivationStatusError); - public ActivationMetricTracker DirectoryRegistrationFailed(Exception? exception, bool cancellationRequested) => WithStatus(exception is null + public void DirectoryRegistrationFailed(Exception? exception, bool cancellationRequested) => SetStatus(exception is null ? CatalogInstruments.ActivationStatusDuplicate : cancellationRequested ? CatalogInstruments.ActivationStatusCanceled : CatalogInstruments.ActivationStatusDirectoryError); - public ActivationMetricTracker Canceled() => WithStatus(CatalogInstruments.ActivationStatusCanceled); + public void Canceled() => SetStatus(CatalogInstruments.ActivationStatusCanceled); public void Record() { @@ -126,11 +126,16 @@ public void Record() return; } - var stopwatch = _stopwatch; - CatalogInstruments.OnActivationCompleted(stopwatch.Elapsed, _status, _usesDirectory); + CatalogInstruments.OnActivationCompleted(_stopwatch.Elapsed, _status, _usesDirectory); } - private ActivationMetricTracker WithStatus(string status) => _status is null ? this : new(_stopwatch, _usesDirectory, status); + private void SetStatus(string status) + { + if (_status is not null) + { + _status = status; + } + } } private readonly struct DeactivationMetricTracker @@ -1735,7 +1740,6 @@ private async Task ActivateAsync(Dictionary? requestContextData, return; } - var metrics = activationMetrics; _activationActivity?.AddEvent(new ActivityEvent("activation-start")); try { @@ -1847,7 +1851,7 @@ private async Task ActivateAsync(Dictionary? requestContextData, if (!success) { Deactivate(new(DeactivationReasonCode.DirectoryFailure, registrationException, "Failed to register activation in grain directory.")); - metrics = metrics.DirectoryRegistrationFailed(registrationException, cancellationToken.IsCancellationRequested); + activationMetrics.DirectoryRegistrationFailed(registrationException, cancellationToken.IsCancellationRequested); // Activation failed. if (registrationException is not null) @@ -1936,7 +1940,7 @@ private async Task ActivateAsync(Dictionary? requestContextData, DeactivationReason.ReasonCode, DeactivationReason.Description, ForwardingAddress); _activationActivity?.Dispose(); _activationActivity = null; - metrics = metrics.Canceled(); + activationMetrics.Canceled(); return; } @@ -1961,12 +1965,12 @@ private async Task ActivateAsync(Dictionary? requestContextData, GrainLifecycleEvents.EmitActivated(this); LogFinishedActivatingGrain(_shared.Logger, this); - metrics = metrics.Succeeded(); + activationMetrics.Succeeded(); } catch (Exception exception) { CatalogInstruments.OnActivationFailedToActivate(); - metrics = metrics.Failed(cancellationToken.IsCancellationRequested); + activationMetrics.Failed(cancellationToken.IsCancellationRequested); var sourceException = (exception as OrleansLifecycleCanceledException)?.InnerException ?? exception; LogErrorActivatingGrain(_shared.Logger, sourceException, this); if (!cancellationToken.IsCancellationRequested) @@ -1983,7 +1987,7 @@ private async Task ActivateAsync(Dictionary? requestContextData, catch (Exception exception) { LogActivationFailed(_shared.Logger, exception, this); - metrics = metrics.Failed(cancellationToken.IsCancellationRequested); + activationMetrics.Failed(cancellationToken.IsCancellationRequested); Deactivate(new(DeactivationReasonCode.ApplicationError, exception, "Failed to activate grain."), CancellationToken.None); SetActivityError(_activationActivity, ActivityErrorEvents.ActivationError); _activationActivity?.Dispose(); @@ -1991,7 +1995,7 @@ private async Task ActivateAsync(Dictionary? requestContextData, } finally { - metrics.Record(); + activationMetrics.Record(); _workSignal.Signal(); } } diff --git a/src/Orleans.Runtime/GrainDirectory/GrainLocator.cs b/src/Orleans.Runtime/GrainDirectory/GrainLocator.cs index 06a97fcb26..e17ab13388 100644 --- a/src/Orleans.Runtime/GrainDirectory/GrainLocator.cs +++ b/src/Orleans.Runtime/GrainDirectory/GrainLocator.cs @@ -27,17 +27,17 @@ public GrainLocator(GrainLocatorResolver grainLocatorResolver) try { var result = await grainLocator.Register(address, previousRegistration); - metrics.Succeeded(); + metrics.RecordSucceeded(); return result; } catch (OperationCanceledException) { - metrics.Canceled(); + metrics.RecordCanceled(); throw; } catch { - metrics.Failed(); + metrics.RecordFailed(); throw; } } @@ -78,11 +78,11 @@ public static RegistrationMetricTracker Start(IGrainLocator grainLocator) : default; } - public void Succeeded() => Record(DirectoryInstruments.RegistrationStatusSuccess); + public void RecordSucceeded() => Record(DirectoryInstruments.RegistrationStatusSuccess); - public void Canceled() => Record(DirectoryInstruments.RegistrationStatusCanceled); + public void RecordCanceled() => Record(DirectoryInstruments.RegistrationStatusCanceled); - public void Failed() => Record(DirectoryInstruments.RegistrationStatusError); + public void RecordFailed() => Record(DirectoryInstruments.RegistrationStatusError); private void Record(string status) { @@ -91,8 +91,7 @@ private void Record(string status) return; } - var stopwatch = _stopwatch; - DirectoryInstruments.OnRegistrationCompleted(stopwatch.Elapsed, _locator, status); + DirectoryInstruments.OnRegistrationCompleted(_stopwatch.Elapsed, _locator, status); } } diff --git a/test/Orleans.Core.Tests/Runtime/CatalogInstrumentsTests.cs b/test/Orleans.Core.Tests/Runtime/CatalogInstrumentsTests.cs index 4d6f8cddaa..7f5a6a5d93 100644 --- a/test/Orleans.Core.Tests/Runtime/CatalogInstrumentsTests.cs +++ b/test/Orleans.Core.Tests/Runtime/CatalogInstrumentsTests.cs @@ -10,20 +10,20 @@ public class CatalogInstrumentsTests [Fact, TestCategory("BVT"), TestCategory("Runtime")] public void ActivationLifecycleLatencyMetrics_AreHistograms() { - Instrument activationDurationInstrument = null!; + Instrument activationLatencyInstrument = null!; Instrument deactivationLatencyInstrument = null!; - var activationDurationMeasurement = 0d; + var activationLatencyMeasurement = 0d; var deactivationLatencyMeasurement = 0d; using var listener = new MeterListener(); listener.InstrumentPublished = (instrument, meterListener) => { - if (instrument.Name is InstrumentNames.CATALOG_ACTIVATION_DURATION or InstrumentNames.CATALOG_DEACTIVATION_LATENCY) + if (instrument.Name is InstrumentNames.CATALOG_ACTIVATION_LATENCY or InstrumentNames.CATALOG_DEACTIVATION_LATENCY) { meterListener.EnableMeasurementEvents(instrument); - if (instrument.Name == InstrumentNames.CATALOG_ACTIVATION_DURATION) + if (instrument.Name == InstrumentNames.CATALOG_ACTIVATION_LATENCY) { - activationDurationInstrument = instrument; + activationLatencyInstrument = instrument; } else { @@ -34,9 +34,9 @@ public void ActivationLifecycleLatencyMetrics_AreHistograms() listener.SetMeasurementEventCallback((instrument, measurement, tags, state) => { - if (instrument.Name == InstrumentNames.CATALOG_ACTIVATION_DURATION) + if (instrument.Name == InstrumentNames.CATALOG_ACTIVATION_LATENCY) { - activationDurationMeasurement = measurement; + activationLatencyMeasurement = measurement; } else if (instrument.Name == InstrumentNames.CATALOG_DEACTIVATION_LATENCY) { @@ -49,11 +49,11 @@ public void ActivationLifecycleLatencyMetrics_AreHistograms() CatalogInstruments.OnActivationCompleted(TimeSpan.FromMilliseconds(12), CatalogInstruments.ActivationStatusSuccess, usesDirectory: true); CatalogInstruments.OnDeactivationCompleted(TimeSpan.FromMilliseconds(34), CatalogInstruments.DeactivationViaCollection); - Assert.IsType>(activationDurationInstrument); + Assert.IsType>(activationLatencyInstrument); Assert.IsType>(deactivationLatencyInstrument); - Assert.Equal("ms", activationDurationInstrument.Unit); + Assert.Equal("ms", activationLatencyInstrument.Unit); Assert.Equal("ms", deactivationLatencyInstrument.Unit); - Assert.Equal(12, activationDurationMeasurement); + Assert.Equal(12, activationLatencyMeasurement); Assert.Equal(34, deactivationLatencyMeasurement); } } From 2302fb0289887289d4f3a5fdf87af1f855f8e50e Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 29 May 2026 10:03:24 -0700 Subject: [PATCH 6/7] refactor(runtime): move activation metric tracker Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Diagnostics/Metrics/CatalogInstruments.cs | 53 ++++++++++++++++ src/Orleans.Runtime/Catalog/ActivationData.cs | 61 ++----------------- 2 files changed, 57 insertions(+), 57 deletions(-) diff --git a/src/Orleans.Core/Diagnostics/Metrics/CatalogInstruments.cs b/src/Orleans.Core/Diagnostics/Metrics/CatalogInstruments.cs index 1df654f04d..612944ed07 100644 --- a/src/Orleans.Core/Diagnostics/Metrics/CatalogInstruments.cs +++ b/src/Orleans.Core/Diagnostics/Metrics/CatalogInstruments.cs @@ -25,6 +25,59 @@ internal static class CatalogInstruments internal const string DeactivationViaMigration = "migration"; internal const string DeactivationViaUnknown = "unknown"; + internal struct ActivationMetricTracker + { + private readonly ValueStopwatch _stopwatch; + private readonly bool _usesDirectory; + private string? _status; + + private ActivationMetricTracker(ValueStopwatch stopwatch, bool usesDirectory, string status) + { + _stopwatch = stopwatch; + _usesDirectory = usesDirectory; + _status = status; + } + + public static ActivationMetricTracker Start(bool usesDirectory) + { + return ActivationLatencyEnabled + ? new(ValueStopwatch.StartNew(), usesDirectory, ActivationStatusError) + : default; + } + + public void Succeeded() => SetStatus(ActivationStatusSuccess); + + public void Failed(bool cancellationRequested) => SetStatus(cancellationRequested + ? ActivationStatusCanceled + : ActivationStatusError); + + public void DirectoryRegistrationFailed(Exception? exception, bool cancellationRequested) => SetStatus(exception is null + ? ActivationStatusDuplicate + : cancellationRequested + ? ActivationStatusCanceled + : ActivationStatusDirectoryError); + + public void Canceled() => SetStatus(ActivationStatusCanceled); + + public void Record() + { + if (_status is null) + { + return; + } + + OnActivationCompleted(_stopwatch.Elapsed, _status, _usesDirectory); + } + + private void SetStatus(string status) + { + if (_status is not null) + { + _status = status; + } + } + } + internal static Counter ActivationFailedToActivate = Instruments.Meter.CreateCounter(InstrumentNames.CATALOG_ACTIVATION_FAILED_TO_ACTIVATE); internal static Counter ActivationCollections = Instruments.Meter.CreateCounter(InstrumentNames.CATALOG_ACTIVATION_COLLECTION_NUMBER_OF_COLLECTIONS); diff --git a/src/Orleans.Runtime/Catalog/ActivationData.cs b/src/Orleans.Runtime/Catalog/ActivationData.cs index 9d858ce7fe..4d85248f72 100644 --- a/src/Orleans.Runtime/Catalog/ActivationData.cs +++ b/src/Orleans.Runtime/Catalog/ActivationData.cs @@ -85,59 +85,6 @@ private static class ActivityErrorEvents public const string DehydrateError = "dehydrate-error"; } - private struct ActivationMetricTracker - { - private readonly ValueStopwatch _stopwatch; - private readonly bool _usesDirectory; - private string? _status; - - private ActivationMetricTracker(ValueStopwatch stopwatch, bool usesDirectory, string status) - { - _stopwatch = stopwatch; - _usesDirectory = usesDirectory; - _status = status; - } - - public static ActivationMetricTracker Start(bool usesDirectory) - { - return CatalogInstruments.ActivationLatencyEnabled - ? new(ValueStopwatch.StartNew(), usesDirectory, CatalogInstruments.ActivationStatusError) - : default; - } - - public void Succeeded() => SetStatus(CatalogInstruments.ActivationStatusSuccess); - - public void Failed(bool cancellationRequested) => SetStatus(cancellationRequested - ? CatalogInstruments.ActivationStatusCanceled - : CatalogInstruments.ActivationStatusError); - - public void DirectoryRegistrationFailed(Exception? exception, bool cancellationRequested) => SetStatus(exception is null - ? CatalogInstruments.ActivationStatusDuplicate - : cancellationRequested - ? CatalogInstruments.ActivationStatusCanceled - : CatalogInstruments.ActivationStatusDirectoryError); - - public void Canceled() => SetStatus(CatalogInstruments.ActivationStatusCanceled); - - public void Record() - { - if (_status is null) - { - return; - } - - CatalogInstruments.OnActivationCompleted(_stopwatch.Elapsed, _status, _usesDirectory); - } - - private void SetStatus(string status) - { - if (_status is not null) - { - _status = status; - } - } - } - private readonly struct DeactivationMetricTracker { private readonly ValueStopwatch _stopwatch; @@ -1725,14 +1672,14 @@ public void Rehydrate(IRehydrationContext context) public void Activate(Dictionary? requestContext, CancellationToken cancellationToken) { - var metrics = ActivationMetricTracker.Start(IsUsingGrainDirectory); + var metrics = CatalogInstruments.ActivationMetricTracker.Start(IsUsingGrainDirectory); var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); cts.CancelAfter(_shared.InternalRuntime.CollectionOptions.Value.ActivationTimeout); ScheduleOperation(new Command.Activate(requestContext, cts, metrics)); } - private async Task ActivateAsync(Dictionary? requestContextData, ActivationMetricTracker activationMetrics, CancellationToken cancellationToken) + private async Task ActivateAsync(Dictionary? requestContextData, CatalogInstruments.ActivationMetricTracker activationMetrics, CancellationToken cancellationToken) { if (State != ActivationState.Creating) { @@ -2550,10 +2497,10 @@ public sealed class Deactivate(CancellationTokenSource cts, ActivationState prev public Activity? Activity { get; } = activity; } - public sealed class Activate(Dictionary? requestContext, CancellationTokenSource cts, ActivationMetricTracker metrics) : Command(cts) + public sealed class Activate(Dictionary? requestContext, CancellationTokenSource cts, CatalogInstruments.ActivationMetricTracker metrics) : Command(cts) { public Dictionary? RequestContext { get; } = requestContext; - public ActivationMetricTracker Metrics { get; } = metrics; + public CatalogInstruments.ActivationMetricTracker Metrics { get; } = metrics; } public sealed class Rehydrate(IRehydrationContext context) : Command(new()) From b439a22977692c364b40f71a609dacb192a1b5a5 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 29 May 2026 10:08:22 -0700 Subject: [PATCH 7/7] refactor(runtime): move deactivation metric tracker Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Diagnostics/Metrics/CatalogInstruments.cs | 52 ++++++++++++++++++ src/Orleans.Runtime/Catalog/ActivationData.cs | 54 +------------------ 2 files changed, 53 insertions(+), 53 deletions(-) diff --git a/src/Orleans.Core/Diagnostics/Metrics/CatalogInstruments.cs b/src/Orleans.Core/Diagnostics/Metrics/CatalogInstruments.cs index 612944ed07..de75635a0f 100644 --- a/src/Orleans.Core/Diagnostics/Metrics/CatalogInstruments.cs +++ b/src/Orleans.Core/Diagnostics/Metrics/CatalogInstruments.cs @@ -78,6 +78,58 @@ private void SetStatus(string status) } } + internal readonly struct DeactivationMetricTracker + { + private readonly ValueStopwatch _stopwatch; + private readonly string? _via; + private readonly bool _recorded; + + private DeactivationMetricTracker(ValueStopwatch stopwatch, string via, bool recorded) + { + _stopwatch = stopwatch; + _via = via; + _recorded = recorded; + } + + public static DeactivationMetricTracker Start() + { + return DeactivationLatencyEnabled + ? new(ValueStopwatch.StartNew(), DeactivationViaUnknown, recorded: false) + : default; + } + + public DeactivationMetricTracker Collection() => WithVia(DeactivationViaCollection); + + public DeactivationMetricTracker DeactivateOnIdle() => WithVia(DeactivationViaDeactivateOnIdle); + + public DeactivationMetricTracker DeactivateStuckActivation() => WithVia(DeactivationViaDeactivateStuckActivation); + + public DeactivationMetricTracker Migration() => WithVia(DeactivationViaMigration); + + public DeactivationMetricTracker Record() + { + if (_via is null || _recorded) + { + return this; + } + + OnDeactivationCompleted(_stopwatch.Elapsed, _via); + return new(_stopwatch, _via, recorded: true); + } + + public void RecordIfNeeded() + { + if (_via is null || _recorded) + { + return; + } + + OnDeactivationCompleted(_stopwatch.Elapsed, _via); + } + + private DeactivationMetricTracker WithVia(string via) => _via is null ? this : new(_stopwatch, via, _recorded); + } + internal static Counter ActivationFailedToActivate = Instruments.Meter.CreateCounter(InstrumentNames.CATALOG_ACTIVATION_FAILED_TO_ACTIVATE); internal static Counter ActivationCollections = Instruments.Meter.CreateCounter(InstrumentNames.CATALOG_ACTIVATION_COLLECTION_NUMBER_OF_COLLECTIONS); diff --git a/src/Orleans.Runtime/Catalog/ActivationData.cs b/src/Orleans.Runtime/Catalog/ActivationData.cs index 4d85248f72..b35ed695eb 100644 --- a/src/Orleans.Runtime/Catalog/ActivationData.cs +++ b/src/Orleans.Runtime/Catalog/ActivationData.cs @@ -85,58 +85,6 @@ private static class ActivityErrorEvents public const string DehydrateError = "dehydrate-error"; } - private readonly struct DeactivationMetricTracker - { - private readonly ValueStopwatch _stopwatch; - private readonly string? _via; - private readonly bool _recorded; - - private DeactivationMetricTracker(ValueStopwatch stopwatch, string via, bool recorded) - { - _stopwatch = stopwatch; - _via = via; - _recorded = recorded; - } - - public static DeactivationMetricTracker Start() - { - return CatalogInstruments.DeactivationLatencyEnabled - ? new(ValueStopwatch.StartNew(), CatalogInstruments.DeactivationViaUnknown, recorded: false) - : default; - } - - public DeactivationMetricTracker Collection() => WithVia(CatalogInstruments.DeactivationViaCollection); - - public DeactivationMetricTracker DeactivateOnIdle() => WithVia(CatalogInstruments.DeactivationViaDeactivateOnIdle); - - public DeactivationMetricTracker DeactivateStuckActivation() => WithVia(CatalogInstruments.DeactivationViaDeactivateStuckActivation); - - public DeactivationMetricTracker Migration() => WithVia(CatalogInstruments.DeactivationViaMigration); - - public DeactivationMetricTracker Record() - { - if (_via is null || _recorded) - { - return this; - } - - CatalogInstruments.OnDeactivationCompleted(_stopwatch.Elapsed, _via); - return new(_stopwatch, _via, recorded: true); - } - - public void RecordIfNeeded() - { - if (_via is null || _recorded) - { - return; - } - - CatalogInstruments.OnDeactivationCompleted(_stopwatch.Elapsed, _via); - } - - private DeactivationMetricTracker WithVia(string via) => _via is null ? this : new(_stopwatch, via, _recorded); - } - public ActivationData( GrainAddress grainAddress, Func createWorkItemGroup, @@ -1976,7 +1924,7 @@ private async Task FinishDeactivating(Command.Deactivate deactivateCommand, Canc { using var _ = deactivateCommand.Activity; - var deactivationMetrics = DeactivationMetricTracker.Start(); + var deactivationMetrics = CatalogInstruments.DeactivationMetricTracker.Start(); var migrating = false; var encounteredError = false; try