Skip to content

Add target based scaling support for Netherite #265

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
nytian merged 14 commits into from
Feb 14, 2025
Merged

Add target based scaling support for Netherite #265

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
dca1f9b
initial commit
bachuv May 3, 2023
c65fadf
port the scaling logic to compute a target instead of a scale recomme…
sebastianburckhardt May 5, 2023
bae7c8f
moved target based scaling code to NetheriteTargetScaler
bachuv May 9, 2023
583d2de
adding tests
bachuv May 17, 2023
91ff2ae
updated test name
bachuv May 17, 2023
53023d3
add instanceId to descriptor (#270)
davidmrdavid May 19, 2023
f47ae9c
updated with main
nytian Feb 11, 2025
24fba84
make netherite metrics provider singleton
nytian Feb 11, 2025
ea07ee0
add deleted package reference
nytian Feb 12, 2025
79c6723
add unit tests for target based scaling, and fix 2 bugs found by test…
sebastianburckhardt Feb 13, 2025
add0518
increase version to 3.1.0
nytian Feb 13, 2025
8efde4e
update index
nytian Feb 13, 2025
6b4a5a9
add empty line
nytian Feb 13, 2025
cfe7249
update by comments make loadpublisher class property
nytian Feb 14, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 9 additions & 4 deletions src/DurableTask.Netherite.AzureFunctions/NetheriteProvider.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,8 @@ public NetheriteProvider(

public override string EventSourceName => "DurableTask-Netherite";

NetheriteMetricsProvider metricsProvider;

/// <inheritdoc/>
public async override Task<string> RetrieveSerializedEntityState(EntityId entityId, JsonSerializerSettings serializerSettings)
{
Expand Down Expand Up @@ -130,7 +132,6 @@ public override bool TryGetScaleMonitor(
}
}

#if !NETSTANDARD
public override bool TryGetTargetScaler(
string functionId,
string functionName,
Expand All @@ -139,13 +140,17 @@ public override bool TryGetTargetScaler(
out ITargetScaler targetScaler)
{
ILoadPublisherService loadPublisher = this.Service.GetLoadPublisher();

@sebastianburckhardt sebastianburckhardt Feb 14, 2025

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This should be inside the conditional below, I think.

NetheriteMetricsProvider metricsProvider = this.Service.GetNetheriteMetricsProvider(loadPublisher, this.Settings.EventHubsConnection);

targetScaler = new NetheriteTargetScaler(functionId, metricsProvider, this);
// Target Scaler is created per function id. And they share the same NetheriteMetricsProvider.
if ( this.metricsProvider == null)
{
this.metricsProvider = this.Service.GetNetheriteMetricsProvider(loadPublisher, this.Settings.EventHubsConnection);
}

targetScaler = new NetheriteTargetScaler(functionId, this.metricsProvider, this);

return true;
}
#endif

public class NetheriteScaleMetrics : ScaleMetrics
{
Expand Down
2 changes: 1 addition & 1 deletion src/DurableTask.Netherite/OrchestrationService/NetheriteOrchestrationService.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -255,7 +255,7 @@ public bool TryGetScalingMonitor(out ScalingMonitor monitor)
internal ILoadPublisherService GetLoadPublisher()
{
return string.IsNullOrEmpty(this.Settings.LoadInformationAzureTableName) ?
new AzureBlobLoadPublisher(this.Settings.BlobStorageConnection, this.Settings.HubName)
new AzureBlobLoadPublisher(this.Settings.BlobStorageConnection, this.Settings.HubName, this.Settings.TaskhubParametersFilePath)
Comment thread
nytian marked this conversation as resolved.
: new AzureTableLoadPublisher(this.Settings.TableStorageConnection, this.Settings.LoadInformationAzureTableName, this.Settings.HubName);
}

Expand Down
26 changes: 19 additions & 7 deletions src/DurableTask.Netherite/Scaling/NetheriteMetricsProvider.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,9 @@ public class NetheriteMetricsProvider
readonly ILoadPublisherService loadPublisher;
readonly ConnectionInfo eventHubsConnection;

DateTime lastMetricsQueryTime = DateTime.MinValue;
Metrics metrics = default;

public NetheriteMetricsProvider(
ILoadPublisherService loadPublisher,
ConnectionInfo eventHubsConnection)
Expand All @@ -26,15 +29,24 @@ public NetheriteMetricsProvider(
public virtual async Task<Metrics> GetMetricsAsync()
{
DateTime now = DateTime.UtcNow;
var loadInformation = await this.loadPublisher.QueryAsync(CancellationToken.None).ConfigureAwait(false);
var busy = await this.TaskHubIsIdleAsync(loadInformation).ConfigureAwait(false);

return new Metrics()
// Collect the metrics every 5 seconds to avoid excessive poling.
// If calling this method more frequently, return the cached metrics.
if ( now >= this.lastMetricsQueryTime.AddSeconds(5))
{
LoadInformation = loadInformation,
Busy = busy,
Timestamp = now,
};
var loadInformation = await this.loadPublisher.QueryAsync(CancellationToken.None).ConfigureAwait(false);
var busy = await this.TaskHubIsIdleAsync(loadInformation).ConfigureAwait(false);

this.lastMetricsQueryTime = now;
this.metrics = new Metrics()
{
LoadInformation = loadInformation,
Busy = busy,
Timestamp = now,
};
}

return this.metrics;
}

/// <summary>
Expand Down