diff --git a/src/Orleans.Core/Diagnostics/Metrics/CatalogInstruments.cs b/src/Orleans.Core/Diagnostics/Metrics/CatalogInstruments.cs index 95a6451fc8d..de75635a0fc 100644 --- a/src/Orleans.Core/Diagnostics/Metrics/CatalogInstruments.cs +++ b/src/Orleans.Core/Diagnostics/Metrics/CatalogInstruments.cs @@ -2,15 +2,22 @@ using System.Collections.Generic; using System.Diagnostics.Metrics; -#nullable disable namespace Orleans.Runtime; internal static class CatalogInstruments { - internal const string ActivationOutcomeCanceled = "canceled"; - internal const string ActivationOutcomeDuplicate = "duplicate"; - internal const string ActivationOutcomeFailure = "failure"; - internal const string ActivationOutcomeSuccess = "success"; + private const string MillisecondsUnit = "ms"; + private const string StatusTagName = "status"; + private const string DirectoryTagName = "directory"; + private const string ViaTagName = "via"; + private const string DirectoryEnabled = "enabled"; + private const string DirectoryDisabled = "disabled"; + + internal const string ActivationStatusSuccess = "success"; + internal const string ActivationStatusCanceled = "canceled"; + internal const string ActivationStatusDirectoryError = "directory_error"; + internal const string ActivationStatusDuplicate = "duplicate"; + internal const string ActivationStatusError = "error"; internal const string DeactivationViaCollection = "collection"; internal const string DeactivationViaDeactivateOnIdle = "deactivateOnIdle"; @@ -18,33 +25,130 @@ internal static class CatalogInstruments internal const string DeactivationViaMigration = "migration"; internal const string DeactivationViaUnknown = "unknown"; - internal static Counter ActivationFailedToActivate = Instruments.Meter.CreateCounter(InstrumentNames.CATALOG_ACTIVATION_FAILED_TO_ACTIVATE); + internal struct ActivationMetricTracker + { + private readonly ValueStopwatch _stopwatch; + private readonly bool _usesDirectory; + private string? _status; - internal static Counter ActivationCollections = Instruments.Meter.CreateCounter(InstrumentNames.CATALOG_ACTIVATION_COLLECTION_NUMBER_OF_COLLECTIONS); + private ActivationMetricTracker(ValueStopwatch stopwatch, bool usesDirectory, string status) + { + _stopwatch = stopwatch; + _usesDirectory = usesDirectory; + _status = status; + } - internal static Counter ActivationShutdown = Instruments.Meter.CreateCounter(InstrumentNames.CATALOG_ACTIVATION_SHUTDOWN); + public static ActivationMetricTracker Start(bool usesDirectory) + { + return ActivationLatencyEnabled + ? new(ValueStopwatch.StartNew(), usesDirectory, ActivationStatusError) + : default; + } - internal static void ActivationShutdownViaCollection() => ActivationShutdown.Add(1, new KeyValuePair("via", DeactivationViaCollection)); - internal static void ActivationShutdownViaDeactivateOnIdle() => ActivationShutdown.Add(1, new KeyValuePair("via", DeactivationViaDeactivateOnIdle)); - internal static void ActivationShutdownViaMigration() => ActivationShutdown.Add(1, new KeyValuePair("via", DeactivationViaMigration)); - internal static void ActivationShutdownViaDeactivateStuckActivation() => ActivationShutdown.Add(1, new KeyValuePair("via", DeactivationViaDeactivateStuckActivation)); + public void Succeeded() => SetStatus(ActivationStatusSuccess); - internal static Histogram ActivationLatency = Instruments.Meter.CreateHistogram(InstrumentNames.CATALOG_ACTIVATION_LATENCY, "ms"); - internal static Histogram DeactivationLatency = Instruments.Meter.CreateHistogram(InstrumentNames.CATALOG_DEACTIVATION_LATENCY, "ms"); + public void Failed(bool cancellationRequested) => SetStatus(cancellationRequested + ? ActivationStatusCanceled + : ActivationStatusError); + + public void DirectoryRegistrationFailed(Exception? exception, bool cancellationRequested) => SetStatus(exception is null + ? ActivationStatusDuplicate + : cancellationRequested + ? ActivationStatusCanceled + : ActivationStatusDirectoryError); + + public void Canceled() => SetStatus(ActivationStatusCanceled); + + public void Record() + { + if (_status is null) + { + return; + } + + OnActivationCompleted(_stopwatch.Elapsed, _status, _usesDirectory); + } + + private void SetStatus(string status) + { + if (_status is not null) + { + _status = status; + } + } + } - internal static void OnActivationCompleted(TimeSpan latency, string outcome) + internal readonly struct DeactivationMetricTracker { - if (ActivationLatency.Enabled) + private readonly ValueStopwatch _stopwatch; + private readonly string? _via; + private readonly bool _recorded; + + private DeactivationMetricTracker(ValueStopwatch stopwatch, string via, bool recorded) { - ActivationLatency.Record(latency.TotalMilliseconds, new KeyValuePair("outcome", outcome)); + _stopwatch = stopwatch; + _via = via; + _recorded = recorded; } + + public static DeactivationMetricTracker Start() + { + return DeactivationLatencyEnabled + ? new(ValueStopwatch.StartNew(), DeactivationViaUnknown, recorded: false) + : default; + } + + public DeactivationMetricTracker Collection() => WithVia(DeactivationViaCollection); + + public DeactivationMetricTracker DeactivateOnIdle() => WithVia(DeactivationViaDeactivateOnIdle); + + public DeactivationMetricTracker DeactivateStuckActivation() => WithVia(DeactivationViaDeactivateStuckActivation); + + public DeactivationMetricTracker Migration() => WithVia(DeactivationViaMigration); + + public DeactivationMetricTracker Record() + { + if (_via is null || _recorded) + { + return this; + } + + OnDeactivationCompleted(_stopwatch.Elapsed, _via); + return new(_stopwatch, _via, recorded: true); + } + + public void RecordIfNeeded() + { + if (_via is null || _recorded) + { + return; + } + + OnDeactivationCompleted(_stopwatch.Elapsed, _via); + } + + private DeactivationMetricTracker WithVia(string via) => _via is null ? this : new(_stopwatch, via, _recorded); } + internal static Counter ActivationFailedToActivate = Instruments.Meter.CreateCounter(InstrumentNames.CATALOG_ACTIVATION_FAILED_TO_ACTIVATE); + + internal static Counter ActivationCollections = Instruments.Meter.CreateCounter(InstrumentNames.CATALOG_ACTIVATION_COLLECTION_NUMBER_OF_COLLECTIONS); + + internal static Counter ActivationShutdown = Instruments.Meter.CreateCounter(InstrumentNames.CATALOG_ACTIVATION_SHUTDOWN); + + internal static void ActivationShutdownViaCollection() => OnActivationShutdown(DeactivationViaCollection); + internal static void ActivationShutdownViaDeactivateOnIdle() => OnActivationShutdown(DeactivationViaDeactivateOnIdle); + internal static void ActivationShutdownViaMigration() => OnActivationShutdown(DeactivationViaMigration); + internal static void ActivationShutdownViaDeactivateStuckActivation() => OnActivationShutdown(DeactivationViaDeactivateStuckActivation); + + internal static Histogram DeactivationLatency = Instruments.Meter.CreateHistogram(InstrumentNames.CATALOG_DEACTIVATION_LATENCY, MillisecondsUnit); + internal static bool DeactivationLatencyEnabled => DeactivationLatency.Enabled; + internal static void OnDeactivationCompleted(TimeSpan latency, string via) { if (DeactivationLatency.Enabled) { - DeactivationLatency.Record(latency.TotalMilliseconds, new KeyValuePair("via", via)); + DeactivationLatency.Record(latency.TotalMilliseconds, new KeyValuePair(ViaTagName, via)); } } @@ -54,17 +158,56 @@ internal static void OnDeactivationCompleted(TimeSpan latency, string via) internal static readonly Counter ActivationsCreated = Instruments.Meter.CreateCounter(InstrumentNames.CATALOG_ACTIVATION_CREATED); internal static readonly Counter ActivationsDestroyed = Instruments.Meter.CreateCounter(InstrumentNames.CATALOG_ACTIVATION_DESTROYED); + private static readonly Histogram ActivationLatency = Instruments.Meter.CreateHistogram(InstrumentNames.CATALOG_ACTIVATION_LATENCY, MillisecondsUnit); + internal static bool ActivationLatencyEnabled => ActivationLatency.Enabled; + + internal static ObservableGauge? ActivationCount; - internal static ObservableGauge ActivationCount; - internal static void RegisterActivationCountObserve(Func observeValue) { ActivationCount = Instruments.Meter.CreateObservableGauge(InstrumentNames.CATALOG_ACTIVATION_COUNT, observeValue); } - internal static ObservableGauge ActivationWorkingSet; + internal static ObservableGauge? ActivationWorkingSet; internal static void RegisterActivationWorkingSetObserve(Func observeValue) { ActivationWorkingSet = Instruments.Meter.CreateObservableGauge(InstrumentNames.CATALOG_ACTIVATION_WORKING_SET, observeValue); } + + internal static void OnActivationCompleted(TimeSpan latency, string status, bool usesDirectory) + { + if (ActivationLatency.Enabled) + { + ActivationLatency.Record( + Math.Max(0, latency.TotalMilliseconds), + [ + new KeyValuePair(StatusTagName, status), + new KeyValuePair(DirectoryTagName, usesDirectory ? DirectoryEnabled : DirectoryDisabled) + ]); + } + } + + internal static void OnActivationFailedToActivate() + { + if (ActivationFailedToActivate.Enabled) + { + ActivationFailedToActivate.Add(1); + } + } + + internal static void OnActivationConcurrentRegistrationAttempt() + { + if (ActivationConcurrentRegistrationAttempts.Enabled) + { + ActivationConcurrentRegistrationAttempts.Add(1); + } + } + + private static void OnActivationShutdown(string via) + { + if (ActivationShutdown.Enabled) + { + ActivationShutdown.Add(1, new KeyValuePair(ViaTagName, via)); + } + } } diff --git a/src/Orleans.Core/Diagnostics/Metrics/DirectoryInstruments.cs b/src/Orleans.Core/Diagnostics/Metrics/DirectoryInstruments.cs index 3028827f8ba..4f1b3ea4fce 100644 --- a/src/Orleans.Core/Diagnostics/Metrics/DirectoryInstruments.cs +++ b/src/Orleans.Core/Diagnostics/Metrics/DirectoryInstruments.cs @@ -1,13 +1,21 @@ using System; +using System.Collections.Generic; using System.Collections.Immutable; using System.Diagnostics.Metrics; using System.Threading; -#nullable disable namespace Orleans.Runtime; internal static class DirectoryInstruments { + private const string MillisecondsUnit = "ms"; + private const string StatusTagName = "status"; + private const string LocatorTagName = "locator"; + + internal const string RegistrationStatusSuccess = "success"; + internal const string RegistrationStatusCanceled = "canceled"; + internal const string RegistrationStatusError = "error"; + private static ImmutableArray CacheSizeObservers = []; internal static readonly Counter LookupsLocalIssued = Instruments.Meter.CreateCounter(InstrumentNames.DIRECTORY_LOOKUPS_LOCAL_ISSUED); @@ -31,7 +39,7 @@ internal static class DirectoryInstruments internal static readonly Histogram RangeRecoveryDuration = Instruments.Meter.CreateHistogram(InstrumentNames.DIRECTORY_RANGE_RECOVERY_DURATION); internal static readonly Histogram RangeLockHeldDuration = Instruments.Meter.CreateHistogram(InstrumentNames.DIRECTORY_RANGE_LOCK_HELD_DURATION); - internal static ObservableGauge DirectoryPartitionSize; + internal static ObservableGauge? DirectoryPartitionSize; internal static void RegisterDirectoryPartitionSizeObserve(Func observeValue) { DirectoryPartitionSize = Instruments.Meter.CreateObservableGauge(InstrumentNames.DIRECTORY_PARTITION_SIZE, observeValue); @@ -47,25 +55,25 @@ internal static IDisposable RegisterCacheSizeObserve(Func observeValue) return registration; } - internal static ObservableGauge RingSize; + internal static ObservableGauge? RingSize; internal static void RegisterRingSizeObserve(Func observeValue) { RingSize = Instruments.Meter.CreateObservableGauge(InstrumentNames.DIRECTORY_RING_RINGSIZE, observeValue); } - internal static ObservableGauge MyPortionRingDistance; + internal static ObservableGauge? MyPortionRingDistance; internal static void RegisterMyPortionRingDistanceObserve(Func observeValue) { MyPortionRingDistance = Instruments.Meter.CreateObservableGauge(InstrumentNames.DIRECTORY_RING_MYPORTION_RINGDISTANCE, observeValue); } - internal static ObservableGauge MyPortionRingPercentage; + internal static ObservableGauge? MyPortionRingPercentage; internal static void RegisterMyPortionRingPercentageObserve(Func observeValue) { MyPortionRingPercentage = Instruments.Meter.CreateObservableGauge(InstrumentNames.DIRECTORY_RING_MYPORTION_RINGPERCENTAGE, observeValue); } - internal static ObservableGauge MyPortionAverageRingPercentage; + internal static ObservableGauge? MyPortionAverageRingPercentage; internal static void RegisterMyPortionAverageRingPercentageObserve(Func observeValue) { MyPortionAverageRingPercentage = Instruments.Meter.CreateObservableGauge(InstrumentNames.DIRECTORY_RING_MYPORTION_AVERAGERINGPERCENTAGE, observeValue); @@ -75,6 +83,9 @@ internal static void RegisterMyPortionAverageRingPercentageObserve(Func o internal static readonly Counter RegistrationsSingleActLocal = Instruments.Meter.CreateCounter(InstrumentNames.DIRECTORY_REGISTRATIONS_SINGLE_ACT_LOCAL); internal static readonly Counter RegistrationsSingleActRemoteSent = Instruments.Meter.CreateCounter(InstrumentNames.DIRECTORY_REGISTRATIONS_SINGLE_ACT_REMOTE_SENT); internal static readonly Counter RegistrationsSingleActRemoteReceived = Instruments.Meter.CreateCounter(InstrumentNames.DIRECTORY_REGISTRATIONS_SINGLE_ACT_REMOTE_RECEIVED); + internal static readonly Counter Registrations = Instruments.Meter.CreateCounter(InstrumentNames.DIRECTORY_REGISTRATIONS); + internal static readonly Histogram RegistrationDuration = Instruments.Meter.CreateHistogram(InstrumentNames.DIRECTORY_REGISTRATION_DURATION, MillisecondsUnit); + internal static bool RegistrationMetricsEnabled => Registrations.Enabled || RegistrationDuration.Enabled; internal static readonly Counter UnregistrationsIssued = Instruments.Meter.CreateCounter(InstrumentNames.DIRECTORY_UNREGISTRATIONS_ISSUED); internal static readonly Counter UnregistrationsLocal = Instruments.Meter.CreateCounter(InstrumentNames.DIRECTORY_UNREGISTRATIONS_LOCAL); internal static readonly Counter UnregistrationsRemoteSent = Instruments.Meter.CreateCounter(InstrumentNames.DIRECTORY_UNREGISTRATIONS_REMOTE_SENT); @@ -96,7 +107,7 @@ private static int ObserveCacheSize() private sealed class CacheSizeObserverRegistration : IDisposable { - private Func observeValue; + private Func? observeValue; public CacheSizeObserverRegistration(Func observeValue) { @@ -117,4 +128,29 @@ public void Dispose() } } } + + internal static void OnRegistrationCompleted(TimeSpan latency, string locator, string status) + { + if (!RegistrationMetricsEnabled) + { + return; + } + + var tags = CreateRegistrationTags(locator, status); + if (Registrations.Enabled) + { + Registrations.Add(1, tags); + } + + if (RegistrationDuration.Enabled) + { + RegistrationDuration.Record(Math.Max(0, latency.TotalMilliseconds), tags); + } + } + + private static KeyValuePair[] CreateRegistrationTags(string locator, string status) => + [ + new(LocatorTagName, locator), + new(StatusTagName, status) + ]; } diff --git a/src/Orleans.Core/Diagnostics/Metrics/InstrumentNames.cs b/src/Orleans.Core/Diagnostics/Metrics/InstrumentNames.cs index bf740eeaa9a..91f8f0cf8d4 100644 --- a/src/Orleans.Core/Diagnostics/Metrics/InstrumentNames.cs +++ b/src/Orleans.Core/Diagnostics/Metrics/InstrumentNames.cs @@ -72,6 +72,8 @@ internal static class InstrumentNames public const string DIRECTORY_REGISTRATIONS_SINGLE_ACT_LOCAL = "orleans-directory-registrations-single-act-local"; public const string DIRECTORY_REGISTRATIONS_SINGLE_ACT_REMOTE_SENT = "orleans-directory-registrations-single-act-remote-sent"; public const string DIRECTORY_REGISTRATIONS_SINGLE_ACT_REMOTE_RECEIVED = "orleans-directory-registrations-single-act-remote-received"; + public const string DIRECTORY_REGISTRATIONS = "orleans-directory-registrations"; + public const string DIRECTORY_REGISTRATION_DURATION = "orleans-directory-registration-duration"; public const string DIRECTORY_UNREGISTRATIONS_ISSUED = "orleans-directory-unregistrations-issued"; public const string DIRECTORY_UNREGISTRATIONS_LOCAL = "orleans-directory-unregistrations-local"; public const string DIRECTORY_UNREGISTRATIONS_REMOTE_SENT = "orleans-directory-unregistrations-remote-sent"; diff --git a/src/Orleans.Runtime/Catalog/ActivationData.cs b/src/Orleans.Runtime/Catalog/ActivationData.cs index 8608fc5919d..b35ed695ebc 100644 --- a/src/Orleans.Runtime/Catalog/ActivationData.cs +++ b/src/Orleans.Runtime/Catalog/ActivationData.cs @@ -1265,7 +1265,7 @@ async Task ProcessOperationsAsync() RehydrateInternal(command.Context); break; case Command.Activate command: - await ActivateAsync(command.RequestContext, command.CancellationToken).SuppressThrowing(); + await ActivateAsync(command.RequestContext, command.Metrics, command.CancellationToken).SuppressThrowing(); break; case Command.Deactivate command: await FinishDeactivating(command, command.CancellationToken).SuppressThrowing(); @@ -1620,13 +1620,14 @@ public void Rehydrate(IRehydrationContext context) public void Activate(Dictionary? requestContext, CancellationToken cancellationToken) { + var metrics = CatalogInstruments.ActivationMetricTracker.Start(IsUsingGrainDirectory); var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); cts.CancelAfter(_shared.InternalRuntime.CollectionOptions.Value.ActivationTimeout); - ScheduleOperation(new Command.Activate(requestContext, cts)); + ScheduleOperation(new Command.Activate(requestContext, cts, metrics)); } - private async Task ActivateAsync(Dictionary? requestContextData, CancellationToken cancellationToken) + private async Task ActivateAsync(Dictionary? requestContextData, CatalogInstruments.ActivationMetricTracker activationMetrics, CancellationToken cancellationToken) { if (State != ActivationState.Creating) { @@ -1634,8 +1635,6 @@ private async Task ActivateAsync(Dictionary? requestContextData, return; } - var activationStopwatch = ValueStopwatch.StartNew(); - var activationOutcome = CatalogInstruments.ActivationOutcomeSuccess; _activationActivity?.AddEvent(new ActivityEvent("activation-start")); try { @@ -1709,7 +1708,7 @@ private async Task ActivateAsync(Dictionary? requestContextData, } success = false; - CatalogInstruments.ActivationConcurrentRegistrationAttempts.Add(1); + CatalogInstruments.OnActivationConcurrentRegistrationAttempt(); LogDuplicateActivation( _shared.Logger, Address, @@ -1746,12 +1745,8 @@ private async Task ActivateAsync(Dictionary? requestContextData, } if (!success) { - activationOutcome = DeactivationReason.ReasonCode is DeactivationReasonCode.DuplicateActivation - ? CatalogInstruments.ActivationOutcomeDuplicate - : cancellationToken.IsCancellationRequested - ? CatalogInstruments.ActivationOutcomeCanceled - : CatalogInstruments.ActivationOutcomeFailure; Deactivate(new(DeactivationReasonCode.DirectoryFailure, registrationException, "Failed to register activation in grain directory.")); + activationMetrics.DirectoryRegistrationFailed(registrationException, cancellationToken.IsCancellationRequested); // Activation failed. if (registrationException is not null) @@ -1815,8 +1810,7 @@ private async Task ActivateAsync(Dictionary? requestContextData, { if (cancellationToken.IsCancellationRequested && exception is ObjectDisposedException or OperationCanceledException) { - activationOutcome = CatalogInstruments.ActivationOutcomeCanceled; - CatalogInstruments.ActivationFailedToActivate.Add(1); + CatalogInstruments.OnActivationFailedToActivate(); // This captures the case where user code in OnActivateAsync doesn't use the passed cancellation token // and makes a call that tries to resolve the scoped IServiceProvider or other type that has been disposed because of cancellation, @@ -1841,6 +1835,7 @@ private async Task ActivateAsync(Dictionary? requestContextData, DeactivationReason.ReasonCode, DeactivationReason.Description, ForwardingAddress); _activationActivity?.Dispose(); _activationActivity = null; + activationMetrics.Canceled(); return; } @@ -1865,13 +1860,12 @@ private async Task ActivateAsync(Dictionary? requestContextData, GrainLifecycleEvents.EmitActivated(this); LogFinishedActivatingGrain(_shared.Logger, this); + activationMetrics.Succeeded(); } catch (Exception exception) { - activationOutcome = cancellationToken.IsCancellationRequested - ? CatalogInstruments.ActivationOutcomeCanceled - : CatalogInstruments.ActivationOutcomeFailure; - CatalogInstruments.ActivationFailedToActivate.Add(1); + CatalogInstruments.OnActivationFailedToActivate(); + activationMetrics.Failed(cancellationToken.IsCancellationRequested); var sourceException = (exception as OrleansLifecycleCanceledException)?.InnerException ?? exception; LogErrorActivatingGrain(_shared.Logger, sourceException, this); if (!cancellationToken.IsCancellationRequested) @@ -1887,8 +1881,8 @@ private async Task ActivateAsync(Dictionary? requestContextData, } catch (Exception exception) { - activationOutcome = CatalogInstruments.ActivationOutcomeFailure; LogActivationFailed(_shared.Logger, exception, this); + activationMetrics.Failed(cancellationToken.IsCancellationRequested); Deactivate(new(DeactivationReasonCode.ApplicationError, exception, "Failed to activate grain."), CancellationToken.None); SetActivityError(_activationActivity, ActivityErrorEvents.ActivationError); _activationActivity?.Dispose(); @@ -1896,7 +1890,7 @@ private async Task ActivateAsync(Dictionary? requestContextData, } finally { - CatalogInstruments.OnActivationCompleted(activationStopwatch.Elapsed, activationOutcome); + activationMetrics.Record(); _workSignal.Signal(); } } @@ -1930,9 +1924,7 @@ private async Task FinishDeactivating(Command.Deactivate deactivateCommand, Canc { using var _ = deactivateCommand.Activity; - var deactivationStopwatch = ValueStopwatch.StartNew(); - var deactivationVia = CatalogInstruments.DeactivationViaUnknown; - var deactivationLatencyRecorded = false; + var deactivationMetrics = CatalogInstruments.DeactivationMetricTracker.Start(); var migrating = false; var encounteredError = false; try @@ -2036,22 +2028,22 @@ private async Task FinishDeactivating(Command.Deactivate deactivateCommand, Canc if (IsStuckDeactivating) { - deactivationVia = CatalogInstruments.DeactivationViaDeactivateStuckActivation; + deactivationMetrics = deactivationMetrics.DeactivateStuckActivation(); CatalogInstruments.ActivationShutdownViaDeactivateStuckActivation(); } else if (migrating) { - deactivationVia = CatalogInstruments.DeactivationViaMigration; + deactivationMetrics = deactivationMetrics.Migration(); CatalogInstruments.ActivationShutdownViaMigration(); } else if (_isInWorkingSet) { - deactivationVia = CatalogInstruments.DeactivationViaDeactivateOnIdle; + deactivationMetrics = deactivationMetrics.DeactivateOnIdle(); CatalogInstruments.ActivationShutdownViaDeactivateOnIdle(); } else { - deactivationVia = CatalogInstruments.DeactivationViaCollection; + deactivationMetrics = deactivationMetrics.Collection(); CatalogInstruments.ActivationShutdownViaCollection(); } @@ -2072,8 +2064,7 @@ private async Task FinishDeactivating(Command.Deactivate deactivateCommand, Canc GrainLifecycleEvents.EmitDeactivated(this, DeactivationReason); } - CatalogInstruments.OnDeactivationCompleted(deactivationStopwatch.Elapsed, deactivationVia); - deactivationLatencyRecorded = true; + deactivationMetrics = deactivationMetrics.Record(); // Signal deactivation GetDeactivationCompletionSource().TrySetResult(true); @@ -2081,10 +2072,7 @@ private async Task FinishDeactivating(Command.Deactivate deactivateCommand, Canc } finally { - if (!deactivationLatencyRecorded) - { - CatalogInstruments.OnDeactivationCompleted(deactivationStopwatch.Elapsed, deactivationVia); - } + deactivationMetrics.RecordIfNeeded(); } async ValueTask StartMigrationAsync(DehydrationContextHolder context, IActivationMigrationManager migrationManager, CancellationToken cancellationToken) @@ -2457,9 +2445,10 @@ public sealed class Deactivate(CancellationTokenSource cts, ActivationState prev public Activity? Activity { get; } = activity; } - public sealed class Activate(Dictionary? requestContext, CancellationTokenSource cts) : Command(cts) + public sealed class Activate(Dictionary? requestContext, CancellationTokenSource cts, CatalogInstruments.ActivationMetricTracker metrics) : Command(cts) { public Dictionary? RequestContext { get; } = requestContext; + public CatalogInstruments.ActivationMetricTracker Metrics { get; } = metrics; } public sealed class Rehydrate(IRehydrationContext context) : Command(new()) diff --git a/src/Orleans.Runtime/GrainDirectory/GrainLocator.cs b/src/Orleans.Runtime/GrainDirectory/GrainLocator.cs index 9fef7115150..e17ab13388b 100644 --- a/src/Orleans.Runtime/GrainDirectory/GrainLocator.cs +++ b/src/Orleans.Runtime/GrainDirectory/GrainLocator.cs @@ -1,3 +1,4 @@ +using System; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Threading.Tasks; @@ -19,7 +20,27 @@ public GrainLocator(GrainLocatorResolver grainLocatorResolver) public ValueTask Lookup(GrainId grainId) => GetGrainLocator(grainId.Type).Lookup(grainId); - public Task Register(GrainAddress address, GrainAddress? previousRegistration) => GetGrainLocator(address.GrainId.Type).Register(address, previousRegistration); + public async Task Register(GrainAddress address, GrainAddress? previousRegistration) + { + var grainLocator = GetGrainLocator(address.GrainId.Type); + var metrics = RegistrationMetricTracker.Start(grainLocator); + try + { + var result = await grainLocator.Register(address, previousRegistration); + metrics.RecordSucceeded(); + return result; + } + catch (OperationCanceledException) + { + metrics.RecordCanceled(); + throw; + } + catch + { + metrics.RecordFailed(); + throw; + } + } public Task Unregister(GrainAddress address, UnregistrationCause cause) => GetGrainLocator(address.GrainId.Type).Unregister(address, cause); @@ -31,6 +52,49 @@ public GrainLocator(GrainLocatorResolver grainLocatorResolver) private IGrainLocator GetGrainLocator(GrainType grainType) => _grainLocatorResolver.GetGrainLocator(grainType); + private static string GetLocatorTag(IGrainLocator grainLocator) => grainLocator switch + { + CachedGrainLocator => "cached", + ClientGrainLocator => "client", + DhtGrainLocator => "dht", + _ => "custom" + }; + + private readonly struct RegistrationMetricTracker + { + private readonly ValueStopwatch _stopwatch; + private readonly string? _locator; + + private RegistrationMetricTracker(ValueStopwatch stopwatch, string locator) + { + _stopwatch = stopwatch; + _locator = locator; + } + + public static RegistrationMetricTracker Start(IGrainLocator grainLocator) + { + return DirectoryInstruments.RegistrationMetricsEnabled + ? new(ValueStopwatch.StartNew(), GetLocatorTag(grainLocator)) + : default; + } + + public void RecordSucceeded() => Record(DirectoryInstruments.RegistrationStatusSuccess); + + public void RecordCanceled() => Record(DirectoryInstruments.RegistrationStatusCanceled); + + public void RecordFailed() => Record(DirectoryInstruments.RegistrationStatusError); + + private void Record(string status) + { + if (_locator is null) + { + return; + } + + DirectoryInstruments.OnRegistrationCompleted(_stopwatch.Elapsed, _locator, status); + } + } + public void UpdateCache(GrainId grainId, SiloAddress siloAddress) => GetGrainLocator(grainId.Type).UpdateCache(grainId, siloAddress); public void UpdateCache(GrainAddressCacheUpdate update) diff --git a/test/Orleans.Core.Tests/Runtime/CatalogInstrumentsTests.cs b/test/Orleans.Core.Tests/Runtime/CatalogInstrumentsTests.cs index 2f7e8873b26..7f5a6a5d93b 100644 --- a/test/Orleans.Core.Tests/Runtime/CatalogInstrumentsTests.cs +++ b/test/Orleans.Core.Tests/Runtime/CatalogInstrumentsTests.cs @@ -46,7 +46,7 @@ public void ActivationLifecycleLatencyMetrics_AreHistograms() listener.Start(); - CatalogInstruments.OnActivationCompleted(TimeSpan.FromMilliseconds(12), CatalogInstruments.ActivationOutcomeSuccess); + CatalogInstruments.OnActivationCompleted(TimeSpan.FromMilliseconds(12), CatalogInstruments.ActivationStatusSuccess, usesDirectory: true); CatalogInstruments.OnDeactivationCompleted(TimeSpan.FromMilliseconds(34), CatalogInstruments.DeactivationViaCollection); Assert.IsType>(activationLatencyInstrument);