Skip to content

feat(runtime): add activation metrics #10147

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ReubenBond merged 7 commits into from
May 29, 2026
Merged

feat(runtime): add activation metrics #10147

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
2a429de
fix(runtime): defer ClientDirectory background loop to BecomeActive l…
ReubenBond May 26, 2026
9d547a5
feat(runtime): add silo catalog and directory activation metrics
ReubenBond May 27, 2026
7413825
perf(runtime): gate activation metric tracking
ReubenBond May 29, 2026
a5dcba5
style(runtime): simplify deactivation metric recording
ReubenBond May 29, 2026
accd90b
refactor(runtime): reduce activation metric churn
ReubenBond May 29, 2026
2302fb0
refactor(runtime): move activation metric tracker
ReubenBond May 29, 2026
b439a22
refactor(runtime): move deactivation metric tracker
ReubenBond May 29, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
185 changes: 164 additions & 21 deletions src/Orleans.Core/Diagnostics/Metrics/CatalogInstruments.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,49 +2,153 @@
using System.Collections.Generic;
using System.Diagnostics.Metrics;

#nullable disable
namespace Orleans.Runtime;

internal static class CatalogInstruments
{
internal const string ActivationOutcomeCanceled = "canceled";
internal const string ActivationOutcomeDuplicate = "duplicate";
internal const string ActivationOutcomeFailure = "failure";
internal const string ActivationOutcomeSuccess = "success";
private const string MillisecondsUnit = "ms";
private const string StatusTagName = "status";
private const string DirectoryTagName = "directory";
private const string ViaTagName = "via";
private const string DirectoryEnabled = "enabled";
private const string DirectoryDisabled = "disabled";

internal const string ActivationStatusSuccess = "success";
internal const string ActivationStatusCanceled = "canceled";
internal const string ActivationStatusDirectoryError = "directory_error";
internal const string ActivationStatusDuplicate = "duplicate";
internal const string ActivationStatusError = "error";

internal const string DeactivationViaCollection = "collection";
internal const string DeactivationViaDeactivateOnIdle = "deactivateOnIdle";
internal const string DeactivationViaDeactivateStuckActivation = "deactivateStuckActivation";
internal const string DeactivationViaMigration = "migration";
internal const string DeactivationViaUnknown = "unknown";

internal static Counter<int> ActivationFailedToActivate = Instruments.Meter.CreateCounter<int>(InstrumentNames.CATALOG_ACTIVATION_FAILED_TO_ACTIVATE);
internal struct ActivationMetricTracker
{
private readonly ValueStopwatch _stopwatch;
private readonly bool _usesDirectory;
private string? _status;

internal static Counter<int> ActivationCollections = Instruments.Meter.CreateCounter<int>(InstrumentNames.CATALOG_ACTIVATION_COLLECTION_NUMBER_OF_COLLECTIONS);
private ActivationMetricTracker(ValueStopwatch stopwatch, bool usesDirectory, string status)
{
_stopwatch = stopwatch;
_usesDirectory = usesDirectory;
_status = status;
}

internal static Counter<int> ActivationShutdown = Instruments.Meter.CreateCounter<int>(InstrumentNames.CATALOG_ACTIVATION_SHUTDOWN);
public static ActivationMetricTracker Start(bool usesDirectory)
{
return ActivationLatencyEnabled
? new(ValueStopwatch.StartNew(), usesDirectory, ActivationStatusError)
: default;
}

internal static void ActivationShutdownViaCollection() => ActivationShutdown.Add(1, new KeyValuePair<string, object>("via", DeactivationViaCollection));
internal static void ActivationShutdownViaDeactivateOnIdle() => ActivationShutdown.Add(1, new KeyValuePair<string, object>("via", DeactivationViaDeactivateOnIdle));
internal static void ActivationShutdownViaMigration() => ActivationShutdown.Add(1, new KeyValuePair<string, object>("via", DeactivationViaMigration));
internal static void ActivationShutdownViaDeactivateStuckActivation() => ActivationShutdown.Add(1, new KeyValuePair<string, object>("via", DeactivationViaDeactivateStuckActivation));
public void Succeeded() => SetStatus(ActivationStatusSuccess);

internal static Histogram<double> ActivationLatency = Instruments.Meter.CreateHistogram<double>(InstrumentNames.CATALOG_ACTIVATION_LATENCY, "ms");
internal static Histogram<double> DeactivationLatency = Instruments.Meter.CreateHistogram<double>(InstrumentNames.CATALOG_DEACTIVATION_LATENCY, "ms");
public void Failed(bool cancellationRequested) => SetStatus(cancellationRequested
? ActivationStatusCanceled
: ActivationStatusError);

public void DirectoryRegistrationFailed(Exception? exception, bool cancellationRequested) => SetStatus(exception is null
? ActivationStatusDuplicate
: cancellationRequested
? ActivationStatusCanceled
: ActivationStatusDirectoryError);

public void Canceled() => SetStatus(ActivationStatusCanceled);

public void Record()
{
if (_status is null)
{
return;
}

OnActivationCompleted(_stopwatch.Elapsed, _status, _usesDirectory);
}

private void SetStatus(string status)
{
if (_status is not null)
{
_status = status;
}
}
}

internal static void OnActivationCompleted(TimeSpan latency, string outcome)
internal readonly struct DeactivationMetricTracker
{
if (ActivationLatency.Enabled)
private readonly ValueStopwatch _stopwatch;
private readonly string? _via;
private readonly bool _recorded;

private DeactivationMetricTracker(ValueStopwatch stopwatch, string via, bool recorded)
{
ActivationLatency.Record(latency.TotalMilliseconds, new KeyValuePair<string, object>("outcome", outcome));
_stopwatch = stopwatch;
_via = via;
_recorded = recorded;
}

public static DeactivationMetricTracker Start()
{
return DeactivationLatencyEnabled
? new(ValueStopwatch.StartNew(), DeactivationViaUnknown, recorded: false)
: default;
}

public DeactivationMetricTracker Collection() => WithVia(DeactivationViaCollection);

public DeactivationMetricTracker DeactivateOnIdle() => WithVia(DeactivationViaDeactivateOnIdle);

public DeactivationMetricTracker DeactivateStuckActivation() => WithVia(DeactivationViaDeactivateStuckActivation);

public DeactivationMetricTracker Migration() => WithVia(DeactivationViaMigration);

public DeactivationMetricTracker Record()
{
if (_via is null || _recorded)
{
return this;
}

OnDeactivationCompleted(_stopwatch.Elapsed, _via);
return new(_stopwatch, _via, recorded: true);
}

public void RecordIfNeeded()
{
if (_via is null || _recorded)
{
return;
}

OnDeactivationCompleted(_stopwatch.Elapsed, _via);
}

private DeactivationMetricTracker WithVia(string via) => _via is null ? this : new(_stopwatch, via, _recorded);
}

internal static Counter<int> ActivationFailedToActivate = Instruments.Meter.CreateCounter<int>(InstrumentNames.CATALOG_ACTIVATION_FAILED_TO_ACTIVATE);

internal static Counter<int> ActivationCollections = Instruments.Meter.CreateCounter<int>(InstrumentNames.CATALOG_ACTIVATION_COLLECTION_NUMBER_OF_COLLECTIONS);

internal static Counter<int> ActivationShutdown = Instruments.Meter.CreateCounter<int>(InstrumentNames.CATALOG_ACTIVATION_SHUTDOWN);

internal static void ActivationShutdownViaCollection() => OnActivationShutdown(DeactivationViaCollection);
internal static void ActivationShutdownViaDeactivateOnIdle() => OnActivationShutdown(DeactivationViaDeactivateOnIdle);
internal static void ActivationShutdownViaMigration() => OnActivationShutdown(DeactivationViaMigration);
internal static void ActivationShutdownViaDeactivateStuckActivation() => OnActivationShutdown(DeactivationViaDeactivateStuckActivation);

internal static Histogram<double> DeactivationLatency = Instruments.Meter.CreateHistogram<double>(InstrumentNames.CATALOG_DEACTIVATION_LATENCY, MillisecondsUnit);
internal static bool DeactivationLatencyEnabled => DeactivationLatency.Enabled;

internal static void OnDeactivationCompleted(TimeSpan latency, string via)
{
if (DeactivationLatency.Enabled)
{
DeactivationLatency.Record(latency.TotalMilliseconds, new KeyValuePair<string, object>("via", via));
DeactivationLatency.Record(latency.TotalMilliseconds, new KeyValuePair<string, object?>(ViaTagName, via));
}
}

Expand All @@ -54,17 +158,56 @@ internal static void OnDeactivationCompleted(TimeSpan latency, string via)

internal static readonly Counter<int> ActivationsCreated = Instruments.Meter.CreateCounter<int>(InstrumentNames.CATALOG_ACTIVATION_CREATED);
internal static readonly Counter<int> ActivationsDestroyed = Instruments.Meter.CreateCounter<int>(InstrumentNames.CATALOG_ACTIVATION_DESTROYED);
private static readonly Histogram<double> ActivationLatency = Instruments.Meter.CreateHistogram<double>(InstrumentNames.CATALOG_ACTIVATION_LATENCY, MillisecondsUnit);
internal static bool ActivationLatencyEnabled => ActivationLatency.Enabled;

internal static ObservableGauge<int>? ActivationCount;

internal static ObservableGauge<int> ActivationCount;

internal static void RegisterActivationCountObserve(Func<int> observeValue)
{
ActivationCount = Instruments.Meter.CreateObservableGauge(InstrumentNames.CATALOG_ACTIVATION_COUNT, observeValue);
}

internal static ObservableGauge<int> ActivationWorkingSet;
internal static ObservableGauge<int>? ActivationWorkingSet;
internal static void RegisterActivationWorkingSetObserve(Func<int> observeValue)
{
ActivationWorkingSet = Instruments.Meter.CreateObservableGauge(InstrumentNames.CATALOG_ACTIVATION_WORKING_SET, observeValue);
}

internal static void OnActivationCompleted(TimeSpan latency, string status, bool usesDirectory)
{
if (ActivationLatency.Enabled)
{
ActivationLatency.Record(
Math.Max(0, latency.TotalMilliseconds),
[
new KeyValuePair<string, object?>(StatusTagName, status),
new KeyValuePair<string, object?>(DirectoryTagName, usesDirectory ? DirectoryEnabled : DirectoryDisabled)
]);
}
}

internal static void OnActivationFailedToActivate()
{
if (ActivationFailedToActivate.Enabled)
{
ActivationFailedToActivate.Add(1);
}
}

internal static void OnActivationConcurrentRegistrationAttempt()
{
if (ActivationConcurrentRegistrationAttempts.Enabled)
{
ActivationConcurrentRegistrationAttempts.Add(1);
}
}

private static void OnActivationShutdown(string via)
{
if (ActivationShutdown.Enabled)
{
ActivationShutdown.Add(1, new KeyValuePair<string, object?>(ViaTagName, via));
}
}
}
50 changes: 43 additions & 7 deletions src/Orleans.Core/Diagnostics/Metrics/DirectoryInstruments.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,21 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Diagnostics.Metrics;
using System.Threading;

#nullable disable
namespace Orleans.Runtime;

internal static class DirectoryInstruments
{
private const string MillisecondsUnit = "ms";
private const string StatusTagName = "status";
private const string LocatorTagName = "locator";

internal const string RegistrationStatusSuccess = "success";
internal const string RegistrationStatusCanceled = "canceled";
internal const string RegistrationStatusError = "error";

private static ImmutableArray<CacheSizeObserverRegistration> CacheSizeObservers = [];

internal static readonly Counter<int> LookupsLocalIssued = Instruments.Meter.CreateCounter<int>(InstrumentNames.DIRECTORY_LOOKUPS_LOCAL_ISSUED);
Expand All @@ -31,7 +39,7 @@ internal static class DirectoryInstruments
internal static readonly Histogram<long> RangeRecoveryDuration = Instruments.Meter.CreateHistogram<long>(InstrumentNames.DIRECTORY_RANGE_RECOVERY_DURATION);
internal static readonly Histogram<long> RangeLockHeldDuration = Instruments.Meter.CreateHistogram<long>(InstrumentNames.DIRECTORY_RANGE_LOCK_HELD_DURATION);

internal static ObservableGauge<int> DirectoryPartitionSize;
internal static ObservableGauge<int>? DirectoryPartitionSize;
internal static void RegisterDirectoryPartitionSizeObserve(Func<int> observeValue)
{
DirectoryPartitionSize = Instruments.Meter.CreateObservableGauge<int>(InstrumentNames.DIRECTORY_PARTITION_SIZE, observeValue);
Expand All @@ -47,25 +55,25 @@ internal static IDisposable RegisterCacheSizeObserve(Func<int> observeValue)
return registration;
}

internal static ObservableGauge<int> RingSize;
internal static ObservableGauge<int>? RingSize;
internal static void RegisterRingSizeObserve(Func<int> observeValue)
{
RingSize = Instruments.Meter.CreateObservableGauge<int>(InstrumentNames.DIRECTORY_RING_RINGSIZE, observeValue);
}

internal static ObservableGauge<long> MyPortionRingDistance;
internal static ObservableGauge<long>? MyPortionRingDistance;
internal static void RegisterMyPortionRingDistanceObserve(Func<long> observeValue)
{
MyPortionRingDistance = Instruments.Meter.CreateObservableGauge<long>(InstrumentNames.DIRECTORY_RING_MYPORTION_RINGDISTANCE, observeValue);
}

internal static ObservableGauge<float> MyPortionRingPercentage;
internal static ObservableGauge<float>? MyPortionRingPercentage;
internal static void RegisterMyPortionRingPercentageObserve(Func<float> observeValue)
{
MyPortionRingPercentage = Instruments.Meter.CreateObservableGauge(InstrumentNames.DIRECTORY_RING_MYPORTION_RINGPERCENTAGE, observeValue);
}

internal static ObservableGauge<float> MyPortionAverageRingPercentage;
internal static ObservableGauge<float>? MyPortionAverageRingPercentage;
internal static void RegisterMyPortionAverageRingPercentageObserve(Func<float> observeValue)
{
MyPortionAverageRingPercentage = Instruments.Meter.CreateObservableGauge(InstrumentNames.DIRECTORY_RING_MYPORTION_AVERAGERINGPERCENTAGE, observeValue);
Expand All @@ -75,6 +83,9 @@ internal static void RegisterMyPortionAverageRingPercentageObserve(Func<float> o
internal static readonly Counter<int> RegistrationsSingleActLocal = Instruments.Meter.CreateCounter<int>(InstrumentNames.DIRECTORY_REGISTRATIONS_SINGLE_ACT_LOCAL);
internal static readonly Counter<int> RegistrationsSingleActRemoteSent = Instruments.Meter.CreateCounter<int>(InstrumentNames.DIRECTORY_REGISTRATIONS_SINGLE_ACT_REMOTE_SENT);
internal static readonly Counter<int> RegistrationsSingleActRemoteReceived = Instruments.Meter.CreateCounter<int>(InstrumentNames.DIRECTORY_REGISTRATIONS_SINGLE_ACT_REMOTE_RECEIVED);
internal static readonly Counter<int> Registrations = Instruments.Meter.CreateCounter<int>(InstrumentNames.DIRECTORY_REGISTRATIONS);
internal static readonly Histogram<double> RegistrationDuration = Instruments.Meter.CreateHistogram<double>(InstrumentNames.DIRECTORY_REGISTRATION_DURATION, MillisecondsUnit);
internal static bool RegistrationMetricsEnabled => Registrations.Enabled || RegistrationDuration.Enabled;
internal static readonly Counter<int> UnregistrationsIssued = Instruments.Meter.CreateCounter<int>(InstrumentNames.DIRECTORY_UNREGISTRATIONS_ISSUED);
internal static readonly Counter<int> UnregistrationsLocal = Instruments.Meter.CreateCounter<int>(InstrumentNames.DIRECTORY_UNREGISTRATIONS_LOCAL);
internal static readonly Counter<int> UnregistrationsRemoteSent = Instruments.Meter.CreateCounter<int>(InstrumentNames.DIRECTORY_UNREGISTRATIONS_REMOTE_SENT);
Expand All @@ -96,7 +107,7 @@ private static int ObserveCacheSize()

private sealed class CacheSizeObserverRegistration : IDisposable
{
private Func<int> observeValue;
private Func<int>? observeValue;

public CacheSizeObserverRegistration(Func<int> observeValue)
{
Expand All @@ -117,4 +128,29 @@ public void Dispose()
}
}
}

internal static void OnRegistrationCompleted(TimeSpan latency, string locator, string status)
{
if (!RegistrationMetricsEnabled)
{
return;
}

var tags = CreateRegistrationTags(locator, status);
if (Registrations.Enabled)
{
Registrations.Add(1, tags);
}

if (RegistrationDuration.Enabled)
{
RegistrationDuration.Record(Math.Max(0, latency.TotalMilliseconds), tags);
}
}

private static KeyValuePair<string, object?>[] CreateRegistrationTags(string locator, string status) =>
[
new(LocatorTagName, locator),
new(StatusTagName, status)
];
}
2 changes: 2 additions & 0 deletions src/Orleans.Core/Diagnostics/Metrics/InstrumentNames.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,8 @@ internal static class InstrumentNames
public const string DIRECTORY_REGISTRATIONS_SINGLE_ACT_LOCAL = "orleans-directory-registrations-single-act-local";
public const string DIRECTORY_REGISTRATIONS_SINGLE_ACT_REMOTE_SENT = "orleans-directory-registrations-single-act-remote-sent";
public const string DIRECTORY_REGISTRATIONS_SINGLE_ACT_REMOTE_RECEIVED = "orleans-directory-registrations-single-act-remote-received";
public const string DIRECTORY_REGISTRATIONS = "orleans-directory-registrations";
public const string DIRECTORY_REGISTRATION_DURATION = "orleans-directory-registration-duration";
public const string DIRECTORY_UNREGISTRATIONS_ISSUED = "orleans-directory-unregistrations-issued";
public const string DIRECTORY_UNREGISTRATIONS_LOCAL = "orleans-directory-unregistrations-local";
public const string DIRECTORY_UNREGISTRATIONS_REMOTE_SENT = "orleans-directory-unregistrations-remote-sent";
Expand Down
Loading