Repository navigation
Add SlowConsumingPressureMonitor #2873
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 7 commits
378d76b
300db7c
9372367
6908027
6562a19
96f7de0
a728bca
bfcc1fb
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,67 @@ | ||
| using Orleans.Runtime; | ||
| using System; | ||
| using System.Collections.Generic; | ||
| using System.Linq; | ||
| using System.Text; | ||
| using System.Threading.Tasks; | ||
|
|
||
| namespace Orleans.ServiceBus.Providers | ||
| { | ||
| /// <summary> | ||
| /// Aggregated cache pressure monitor | ||
| /// </summary> | ||
| public class AggregatedCachePressureMonitor : List<ICachePressureMonitor>, ICachePressureMonitor | ||
| { | ||
| private bool isUnderPressure; | ||
| private Logger logger; | ||
|
|
||
| /// <summary> | ||
| /// Constructor | ||
| /// </summary> | ||
| /// <param name="logger"></param> | ||
| public AggregatedCachePressureMonitor(Logger logger) | ||
| { | ||
| this.isUnderPressure = false; | ||
| this.logger = logger.GetSubLogger(this.GetType().Name); | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Record cache pressure to every monitor in this aggregated cache monitor group | ||
| /// </summary> | ||
| /// <param name="cachePressureContribution"></param> | ||
| public void RecordCachePressureContribution(double cachePressureContribution) | ||
| { | ||
| this.ForEach(monitor => | ||
| { | ||
| monitor.RecordCachePressureContribution(cachePressureContribution); | ||
| }); | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Add one monitor to this aggregated cache monitor group | ||
| /// </summary> | ||
| /// <param name="monitor"></param> | ||
| public void AddCachePressureMonitor(ICachePressureMonitor monitor) | ||
| { | ||
| this.Add(monitor); | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// If any mornitor in this aggregated cache monitor group is under pressure, then return true | ||
| /// </summary> | ||
| /// <param name="utcNow"></param> | ||
| /// <returns></returns> | ||
| public bool IsUnderPressure(DateTime utcNow) | ||
| { | ||
| bool underPressure = this.Any(monitor => monitor.IsUnderPressure(utcNow)); | ||
| if (this.isUnderPressure != underPressure) | ||
| { | ||
| this.isUnderPressure = underPressure; | ||
| logger.Info(this.isUnderPressure | ||
| ? $"Ingesting messages too fast. Throttling message reading." | ||
| : $"Message ingestion is healthy."); | ||
| } | ||
| return underPressure; | ||
| } | ||
| } | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,93 @@ | ||
| using Orleans.Runtime; | ||
| using System; | ||
| using System.Collections.Generic; | ||
| using System.Linq; | ||
| using System.Text; | ||
| using System.Threading.Tasks; | ||
|
|
||
| namespace Orleans.ServiceBus.Providers | ||
| { | ||
| /// <summary> | ||
| /// Cache pressure monitor whose back pressure algorithm is based on averaging pressure value | ||
| /// over all pressure contribution | ||
| /// </summary> | ||
| public class AveragingCachePressureMonitor : ICachePressureMonitor | ||
| { | ||
| /// <summary> | ||
| /// Default flow control threshold | ||
| /// </summary> | ||
| public static readonly double DefaultThreshold = 1 / 3; | ||
| private static readonly TimeSpan checkPeriod = TimeSpan.FromSeconds(2); | ||
| private readonly Logger logger; | ||
|
|
||
| private double accumulatedCachePressure; | ||
| private double cachePressureContributionCount; | ||
| private DateTime nextCheckedTime; | ||
| private bool isUnderPressure; | ||
| private double flowControlThreshold; | ||
|
|
||
| /// <summary> | ||
| /// Constructor | ||
| /// </summary> | ||
| /// <param name="logger"></param> | ||
| public AveragingCachePressureMonitor(Logger logger) | ||
| :this(DefaultThreshold, logger) | ||
| { } | ||
|
|
||
| /// <summary> | ||
| /// Contructor | ||
| /// </summary> | ||
| /// <param name="flowControlThreshold"></param> | ||
| /// <param name="logger"></param> | ||
| public AveragingCachePressureMonitor(double flowControlThreshold, Logger logger) | ||
| { | ||
| this.flowControlThreshold = flowControlThreshold; | ||
| this.logger = logger.GetSubLogger(this.GetType().Name); | ||
| nextCheckedTime = DateTime.MinValue; | ||
| isUnderPressure = false; | ||
| } | ||
|
|
||
| public void RecordCachePressureContribution(double cachePressureContribution) | ||
| { | ||
| // Weight unhealthy contributions thrice as much as healthy ones. | ||
| // This is a crude compensation for the fact that healthy consumers wil consume more often than unhealthy ones. | ||
| double weight = cachePressureContribution < flowControlThreshold ? 1.0 : 3.0; | ||
| accumulatedCachePressure += cachePressureContribution * weight; | ||
| cachePressureContributionCount += weight; | ||
| } | ||
|
|
||
| public bool IsUnderPressure(DateTime utcNow) | ||
| { | ||
| if (nextCheckedTime < utcNow) | ||
| { | ||
| CalculatePressure(); | ||
| nextCheckedTime = utcNow + checkPeriod; | ||
| } | ||
| return isUnderPressure; | ||
| } | ||
|
|
||
| private void CalculatePressure() | ||
| { | ||
| // if we don't have any contributions, don't change status | ||
| if (cachePressureContributionCount < 0.5) | ||
| { | ||
| // after 5 checks with no contributions, check anyway | ||
| cachePressureContributionCount += 0.1; | ||
| return; | ||
| } | ||
|
|
||
| double pressure = accumulatedCachePressure / cachePressureContributionCount; | ||
| bool wasUnderPressure = isUnderPressure; | ||
| isUnderPressure = pressure > flowControlThreshold; | ||
| // If we changed state, log | ||
| if (isUnderPressure != wasUnderPressure) | ||
| { | ||
| logger.Verbose(isUnderPressure | ||
| ? $"Ingesting messages too fast. Throttling message reading. AccumulatedCachePressure: {accumulatedCachePressure}, Contributions: {cachePressureContributionCount}, AverageCachePressure: {pressure}, Threshold: {flowControlThreshold}" | ||
| : $"Message ingestion is healthy. AccumulatedCachePressure: {accumulatedCachePressure}, Contributions: {cachePressureContributionCount}, AverageCachePressure: {pressure}, Threshold: {flowControlThreshold}"); | ||
| } | ||
| cachePressureContributionCount = 0.0; | ||
| accumulatedCachePressure = 0.0; | ||
| } | ||
| } | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,28 @@ | ||
| using System; | ||
| using System.Collections.Generic; | ||
| using System.Linq; | ||
| using System.Text; | ||
| using System.Threading.Tasks; | ||
|
|
||
| namespace Orleans.ServiceBus.Providers | ||
| { | ||
| /// <summary> | ||
| /// Cache pressure monitor records pressure contribution to the cache, and determine if the cache is under pressure based on its | ||
| /// back pressure algorithm | ||
| /// </summary> | ||
| public interface ICachePressureMonitor | ||
| { | ||
| /// <summary> | ||
| /// Record cache pressure contribution to the monitor | ||
| /// </summary> | ||
| /// <param name="cachePressureContribution"></param> | ||
| void RecordCachePressureContribution(double cachePressureContribution); | ||
|
|
||
| /// <summary> | ||
| /// Determine if the monitor is under pressure | ||
| /// </summary> | ||
| /// <param name="utcNow"></param> | ||
| /// <returns></returns> | ||
| bool IsUnderPressure(DateTime utcNow); | ||
| } | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,111 @@ | ||
| using Orleans.Runtime; | ||
| using System; | ||
| using System.Collections.Generic; | ||
| using System.Linq; | ||
| using System.Text; | ||
| using System.Threading.Tasks; | ||
|
|
||
| namespace Orleans.ServiceBus.Providers | ||
| { | ||
| /// <summary> | ||
| /// Pressure monitor which is in favor of the slow consumer in the cache | ||
| /// </summary> | ||
| public class SlowConsumingPressureMonitor : ICachePressureMonitor | ||
| { | ||
| private static TimeSpan DefaultPressureWindowSize = TimeSpan.FromMinutes(1); | ||
| private const double DefaultFlowControlThreshold = 0.5; | ||
|
|
||
| /// <summary> | ||
| /// PressureWindowSize | ||
| /// </summary> | ||
| public TimeSpan PressureWindowSize { get; set; } | ||
| /// <summary> | ||
| /// FlowControlThreshold | ||
| /// </summary> | ||
| public double FlowControlThreshold { get; set; } | ||
|
|
||
| private readonly Logger logger; | ||
| private double biggestPressureInCurrentWindow; | ||
| private DateTime nextCheckedTime; | ||
| private bool wasUnderPressure; | ||
|
|
||
| /// <summary> | ||
| /// Constructor | ||
| /// </summary> | ||
| /// <param name="logger"></param> | ||
| public SlowConsumingPressureMonitor(Logger logger) | ||
| : this(DefaultFlowControlThreshold, DefaultPressureWindowSize, logger) | ||
| { } | ||
|
|
||
| /// <summary> | ||
| /// Constructor | ||
| /// </summary> | ||
| /// <param name="pressureWindowSize"></param> | ||
| /// <param name="logger"></param> | ||
| public SlowConsumingPressureMonitor(TimeSpan pressureWindowSize, Logger logger) | ||
| : this(DefaultFlowControlThreshold, pressureWindowSize, logger) | ||
| { | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Constructor | ||
| /// </summary> | ||
| /// <param name="flowControlThreshold"></param> | ||
| /// <param name="logger"></param> | ||
| public SlowConsumingPressureMonitor(double flowControlThreshold, Logger logger) | ||
| : this(flowControlThreshold, DefaultPressureWindowSize, logger) | ||
| { | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Constructor | ||
| /// </summary> | ||
| /// <param name="flowControlThreshold"></param> | ||
| /// <param name="pressureWindowSzie"></param> | ||
| /// <param name="logger"></param> | ||
| public SlowConsumingPressureMonitor(double flowControlThreshold, TimeSpan pressureWindowSzie, Logger logger) | ||
| { | ||
| this.FlowControlThreshold = flowControlThreshold; | ||
| this.logger = logger.GetSubLogger(this.GetType().Name); | ||
| this.nextCheckedTime = DateTime.MinValue; | ||
| this.biggestPressureInCurrentWindow = 0; | ||
| this.wasUnderPressure = false; | ||
| this.PressureWindowSize = pressureWindowSzie; | ||
| } | ||
|
|
||
| public void RecordCachePressureContribution(double cachePressureContribution) | ||
| { | ||
| if (cachePressureContribution > this.biggestPressureInCurrentWindow) | ||
| biggestPressureInCurrentWindow = cachePressureContribution; | ||
| } | ||
|
|
||
| public bool IsUnderPressure(DateTime utcNow) | ||
| { | ||
| //if any pressure contribution in current period is bigger than flowControlThreshold | ||
| //we see the cache is under pressure | ||
| bool underPressure = this.biggestPressureInCurrentWindow > this.FlowControlThreshold; | ||
|
|
||
| if (underPressure && !this.wasUnderPressure) | ||
| { | ||
| //if under pressure, extend the nextCheckedTime, make sure wasUnderPressure is true for a whole window | ||
| this.wasUnderPressure = underPressure; | ||
| this.nextCheckedTime = utcNow + this.PressureWindowSize; | ||
| logger.Verbose($"Ingesting messages too fast. Throttling message reading. BiggestPressureInCurrentPeriod: {biggestPressureInCurrentWindow}, Threshold: {FlowControlThreshold}"); | ||
| this.biggestPressureInCurrentWindow = 0; | ||
| } | ||
|
|
||
| if (this.nextCheckedTime < utcNow) | ||
| { | ||
| //at the end of each check period, reset biggestPressureInCurrentPeriod | ||
| this.nextCheckedTime = utcNow + this.PressureWindowSize; | ||
| this.biggestPressureInCurrentWindow = 0; | ||
| //if at the end of the window, pressure clears out, log | ||
| if(this.wasUnderPressure && !underPressure) | ||
| logger.Verbose($"Message ingestion is healthy. BiggestPressureInCurrentPeriod: {biggestPressureInCurrentWindow}, Threshold: {FlowControlThreshold}"); | ||
| this.wasUnderPressure = underPressure; | ||
| } | ||
|
|
||
| return this.wasUnderPressure; | ||
|
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. in this logic, monitor will stay underPressure for a whole window if a slow consumer appears, while biggestPressureInCurrentWindow will be reset to 0. So at the end of this window, if still underPressure, then underPressure stays for another whole window. Otherwise, monitor become notUnderPressure. run through real life case, welcome to run through more scenario with me, and let me know if this algorithm is conservative enough
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. or if you want something even more conservative, such as : isUnderPressure will be true until slow consumer catch up to above threshold, and be healthy for a whole window
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think the logic works. It immediately stops reading if there is a cursor past the threshold, but if currently under pressure or no cursor is past the threshold, it only checks again every window. |
||
| } | ||
| } | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Defaults should probably be public, so users can specify them in configurations if they want to be explicit.