diff --git a/src/Orleans.Core/Configuration/OptionLogger/DefaultOptionsFormatter.cs b/src/Orleans.Core/Configuration/OptionLogger/DefaultOptionsFormatter.cs index 456360a173f..f7fbe4d0748 100644 --- a/src/Orleans.Core/Configuration/OptionLogger/DefaultOptionsFormatter.cs +++ b/src/Orleans.Core/Configuration/OptionLogger/DefaultOptionsFormatter.cs @@ -87,7 +87,7 @@ private IEnumerable FormatProperty(PropertyInfo property) } } - internal class DefaultOptionsFormatterResolver : IOptionFormatterResolver + internal class DefaultOptionsFormatterResolver : IOptionFormatterResolver where T: class, new() { private IOptionsSnapshot optionsSnapshot; diff --git a/src/Orleans.Core/Messaging/RequestInvocationHistory.cs b/src/Orleans.Core/Messaging/RequestInvocationHistory.cs index 90ad9b82450..c345a2cc3ab 100644 --- a/src/Orleans.Core/Messaging/RequestInvocationHistory.cs +++ b/src/Orleans.Core/Messaging/RequestInvocationHistory.cs @@ -4,22 +4,36 @@ namespace Orleans.Runtime { // used for tracking request invocation history for deadlock detection. [Serializable] - internal sealed class RequestInvocationHistory + internal sealed class RequestInvocationHistory : RequestInvocationHistorySummary { public GrainId GrainId { get; private set; } - public ActivationId ActivationId { get; private set; } public string DebugContext { get; private set; } - public RequestInvocationHistory(GrainId grainId, ActivationId activationId, string debugContext) + public RequestInvocationHistory(GrainId grainId, ActivationId activationId, string debugContext) : base(activationId) { this.GrainId = grainId; - this.ActivationId = activationId; DebugContext = debugContext; } public override string ToString() { - return String.Format("RequestInvocationHistory {0}:{1}:{2}", GrainId, ActivationId, DebugContext); + return $"RequestInvocationHistory {GrainId}:{ActivationId}:{DebugContext}"; + } + } + // used for tracking request invocation history for call chain reentrancy + [Serializable] + internal class RequestInvocationHistorySummary + { + public ActivationId ActivationId { get; private set; } + + public RequestInvocationHistorySummary(ActivationId activationId) + { + this.ActivationId = activationId; + } + + public override string ToString() + { + return $"RequestInvocationHistorySummary {ActivationId}"; } } } diff --git a/src/Orleans.Runtime.Legacy/Configuration/LegacyClusterConfigurationExtensions.cs b/src/Orleans.Runtime.Legacy/Configuration/LegacyClusterConfigurationExtensions.cs index 02d281c3bd1..18ec2c7c792 100644 --- a/src/Orleans.Runtime.Legacy/Configuration/LegacyClusterConfigurationExtensions.cs +++ b/src/Orleans.Runtime.Legacy/Configuration/LegacyClusterConfigurationExtensions.cs @@ -185,7 +185,7 @@ private static void AddLegacyClusterConfigurationSupport(IServiceCollection serv var nodeConfig = configuration.GetOrCreateNodeConfigurationForSilo(siloOptions.Value.SiloName); options.ExcludedGrainTypes.AddRange(nodeConfig.ExcludedGrainTypes); }); - + services.AddOptions() .Configure((options, config) => { @@ -273,7 +273,7 @@ private static void AddLegacyClusterConfigurationSupport(IServiceCollection serv { options.IsRunningAsUnitTest = config.IsRunningAsUnitTest; }); - + services.AddOptions() .Configure((options, config) => { diff --git a/src/Orleans.Runtime/Catalog/ActivationData.cs b/src/Orleans.Runtime/Catalog/ActivationData.cs index a5bc74b4c61..d4de86b018e 100644 --- a/src/Orleans.Runtime/Catalog/ActivationData.cs +++ b/src/Orleans.Runtime/Catalog/ActivationData.cs @@ -24,7 +24,7 @@ namespace Orleans.Runtime /// internal class ActivationData : IGrainActivationContext, IActivationData, IInvokable, IDisposable { - // This class is used for activations that have extension invokers. It keeps a dictionary of + // This class is used for activations that have extension invokers. It keeps a dictionary of // invoker objects to use with the activation, and extend the default invoker // defined for the grain class. // Note that in all cases we never have more than one copy of an actual invoker; @@ -34,7 +34,7 @@ private class ExtensionInvoker : IGrainMethodInvoker, IGrainExtensionMap // Because calls to ExtensionInvoker are allways made within the activation context, // we rely on the single-threading guarantee of the runtime and do not protect the map with a lock. private Dictionary> extensionMap; // key is the extension interface ID - + /// /// Try to add an extension for the specific interface ID. /// Fail and return false if there is already an extension for that interface ID. @@ -171,7 +171,7 @@ internal class GrainActivationContextFactory private IGrainMethodInvoker lastInvoker; private IServiceScope serviceScope; private HashSet timers; - + public ActivationData( ActivationAddress addr, string genericArguments, @@ -264,8 +264,6 @@ internal bool TryGetExtensionHandler(Type extensionType, out IGrainExtension res return extensionInvoker != null && extensionInvoker.TryGetExtensionHandler(extensionType, out result); } - public HashSet RunningRequestsSenders { get; } = new HashSet(); - public ISchedulingContext SchedulingContext { get; } public string GrainTypeName @@ -313,14 +311,14 @@ private static void SetGrainActivationContextInScopedServices(IServiceProvider s var contextFactory = sp.GetRequiredService(); contextFactory.Context = context; } - + private Streams.StreamDirectory streamDirectory; internal Streams.StreamDirectory GetStreamDirectory() { return streamDirectory ?? (streamDirectory = new Streams.StreamDirectory()); } - internal bool IsUsingStreams + internal bool IsUsingStreams { get { return streamDirectory != null; } } @@ -343,7 +341,7 @@ GrainReference IActivationData.GrainReference { get { return GrainReference; } } - + public GrainId Identity { get { return Grain; } @@ -435,7 +433,7 @@ public void ResetCollectionTicket() public void SetCollectionTicket(DateTime ticket) { if (ticket == default(DateTime)) throw new ArgumentException("default(DateTime) is disallowed", "ticket"); - if (CollectionTicket != default(DateTime)) + if (CollectionTicket != default(DateTime)) { throw new InvalidOperationException("call ResetCollectionTicket before calling SetCollectionTicket."); } @@ -449,7 +447,7 @@ public void SetCollectionTicket(DateTime ticket) // Currently, the only supported multi-activation grain is one using the StatelessWorkerPlacement strategy. internal bool IsStatelessWorker => this.PlacedUsing is StatelessWorkerPlacement; - + /// /// Returns a value indicating whether or not this placement strategy requires activations to be registered in /// the grain directory. @@ -471,12 +469,6 @@ public void RecordRunning(Message message) // Note: This method is always called while holding lock on this activation, so no need for additional locks here numRunning++; - if (message.Direction != Message.Directions.OneWay - && message.SendingActivation != null - && !message.SendingGrain?.IsClient == true) - { - RunningRequestsSenders.Add(message.SendingActivation); - } if (Running != null) return; @@ -490,7 +482,7 @@ public void ResetRunning(Message message) { // Note: This method is always called while holding lock on this activation, so no need for additional locks here numRunning--; - RunningRequestsSenders.Remove(message.SendingActivation); + if (numRunning == 0) { becameIdle = DateTime.UtcNow; @@ -523,7 +515,7 @@ public void ResetRunning(Message message) /// Increment the number of in-flight messages currently being processed. public void IncrementInFlightCount() { Interlocked.Increment(ref inFlightCount); } - + /// Decrement the number of in-flight messages currently being processed. public void DecrementInFlightCount() { Interlocked.Decrement(ref inFlightCount); } @@ -532,14 +524,14 @@ public void ResetRunning(Message message) /// Decrement the number of messages currently in the process of being received. public void DecrementEnqueuedOnDispatcherCount() { Interlocked.Decrement(ref enqueuedOnDispatcherCount); } - + /// /// grouped by sending activation: responses first, then sorted by id /// private List waiting; - public int WaitingCount - { + public int WaitingCount + { get { return waiting == null ? 0 : waiting.Count; @@ -602,7 +594,7 @@ public EnqueueMessageResult EnqueueMessage(Message message) } /// - /// Check whether this activation is overloaded. + /// Check whether this activation is overloaded. /// Returns LimitExceededException if overloaded, otherwise nullc> /// /// Logger to use for reporting any overflow condition @@ -625,7 +617,7 @@ public LimitExceededException CheckOverloaded(ILogger log) if (maxRequestsHardLimit > 0 && count > maxRequestsHardLimit) // Hard limit { - log.Warn(ErrorCode.Catalog_Reject_ActivationTooManyRequests, + log.Warn(ErrorCode.Catalog_Reject_ActivationTooManyRequests, String.Format("Overload - {0} enqueued requests for activation {1}, exceeding hard limit rejection threshold of {2}", count, this, maxRequestsHardLimit)); @@ -699,7 +691,7 @@ public TimeSpan GetIdleness(DateTime now) { if (now == default(DateTime)) throw new ArgumentException("default(DateTime) is not allowed; Use DateTime.UtcNow instead.", "now"); - + return now - becameIdle; } @@ -803,7 +795,7 @@ public void OnTimerDisposed(IGrainTimer orleansTimerInsideGrain) internal Task WaitForAllTimersToFinish() { lock(this) - { + { if (timers == null) { return Task.CompletedTask; diff --git a/src/Orleans.Runtime/Catalog/Catalog.cs b/src/Orleans.Runtime/Catalog/Catalog.cs index 824b4d9ad34..6af84b183af 100644 --- a/src/Orleans.Runtime/Catalog/Catalog.cs +++ b/src/Orleans.Runtime/Catalog/Catalog.cs @@ -35,7 +35,7 @@ internal class NonExistentActivationException : Exception public NonExistentActivationException() : base("NonExistentActivationException") { } public NonExistentActivationException(string msg) : base(msg) { } - public NonExistentActivationException(string message, Exception innerException) + public NonExistentActivationException(string message, Exception innerException) : base(message, innerException) { } public NonExistentActivationException(string msg, ActivationAddress nonExistentActivation, bool isStatelessWorker) @@ -274,7 +274,7 @@ public List> GetGrainStatistics() // TODO: generic type expansion var grainTypeName = TypeUtils.GetFullName(data.GrainInstanceType); - + Dictionary grains; int n; if (!counts.TryGetValue(grainTypeName, out grains)) @@ -347,8 +347,8 @@ public DetailedGrainReport GetDetailedGrainReport(GrainId grain) } List acts = activations.FindTargets(grain); - report.LocalActivations = acts != null ? - acts.Select(activationData => activationData.ToDetailedString()).ToList() : + report.LocalActivations = acts != null ? + acts.Select(activationData => activationData.ToDetailedString()).ToList() : new List(); return report; } @@ -397,18 +397,18 @@ internal int UnregisterGrainForTesting(GrainId grain) int numActsBefore = acts.Count; foreach (var act in acts) UnregisterMessageTarget(act); - + return numActsBefore; } internal bool CanInterleave(ActivationId running, Message message) { - ActivationData target; - GrainTypeData data; - return TryGetActivationData(running, out target) && - target.GrainInstance != null && - GrainTypeManager.TryGetData(TypeUtils.GetFullName(target.GrainInstanceType), out data) && - (data.IsReentrant || data.MayInterleave((InvokeMethodRequest)message.GetDeserializedBody(this.serializationManager))); + return this.TryGetActivationData(running, out ActivationData target) + && target.GrainInstance != null + && this.GrainTypeManager.TryGetData(TypeUtils.GetFullName(target.GrainInstanceType), out GrainTypeData data) + && (data.IsReentrant + || (message.GetDeserializedBody(this.serializationManager) is InvokeMethodRequest invokeMethodRequest + && data.MayInterleave((invokeMethodRequest)))); } public void GetGrainTypeInfo(int typeCode, out string grainClass, out PlacementStrategy placement, out MultiClusterRegistrationStrategy activationStrategy, string genericArguments = null) @@ -449,7 +449,7 @@ public ActivationData GetOrCreateActivation( { return result; } - + int typeCode = address.Grain.TypeCode; string actualGrainType = null; MultiClusterRegistrationStrategy activationStrategy; @@ -502,7 +502,7 @@ public ActivationData GetOrCreateActivation( CounterStatistic.FindOrCreate(StatisticNames.CATALOG_ACTIVATION_NON_EXISTENT_ACTIVATIONS).Increment(); throw new NonExistentActivationException(msg, address, placement is StatelessWorkerPlacement); } - + SetupActivationInstance(result, grainType, genericArguments); activatedPromise = InitActivation(result, grainType, genericArguments, requestContextData); return result; @@ -714,7 +714,7 @@ private void CreateGrainInstance(string grainTypeName, ActivationData data, stri data.SetupContext(grainTypeData, this.serviceProvider); Grain grain = grainCreator.CreateGrainInstance(data); - + //if grain implements IStreamSubscriptionObserver, then install stream consumer extension on it if(grain is IStreamSubscriptionObserver) InstallStreamConsumerExtension(data, grain as IStreamSubscriptionObserver); @@ -722,7 +722,7 @@ private void CreateGrainInstance(string grainTypeName, ActivationData data, stri grain.Data = data; data.SetGrainInstance(grain); } - + activations.IncrementGrainCounter(grainClassName); if (logger.IsEnabled(LogLevel.Debug)) logger.Debug("CreateGrainInstance {0}{1}", data.Grain, data.ActivationId); @@ -932,11 +932,11 @@ private void DestroyActivationAsync(ActivationData activation, MultiTaskCompleti /// // Overall code flow: // Deactivating state was already set before, in the correct context under lock. - // that means no more new requests will be accepted into this activation and all timer were stopped (no new ticks will be delivered or enqueued) + // that means no more new requests will be accepted into this activation and all timer were stopped (no new ticks will be delivered or enqueued) // Wait for all already scheduled ticks to finish // CallGrainDeactivate // when AsyncDeactivate promise is resolved (NOT when all Deactivate turns are done, which may be orphan tasks): - // Unregister in the directory + // Unregister in the directory // when all AsyncDeactivate turns are done (Dispatcher.OnActivationCompletedRequest): // Set Invalid state // UnregisterMessageTarget -> no new tasks will be enqueue (if an orphan task get enqueud, it is ignored and dropped on the floor). @@ -1003,7 +1003,7 @@ private async void FinishDestroyActivations(List list, int numbe //logger.Info(ErrorCode.Catalog_DestroyActivations_Done, "Starting FinishDestroyActivations #{0} - with {1} Activations.", number, list.Count); // step 3 - UnregisterManyAsync try - { + { List activationsToDeactivate = list. Where((ActivationData d) => d.IsUsingGrainDirectory). Select((ActivationData d) => ActivationAddress.GetAddress(LocalSilo, d.Grain, d.ActivationId)).ToList(); @@ -1039,7 +1039,7 @@ await scheduler.RunOrQueueTask(() => // IMPORTANT: no more awaits and .Ignore after that point. - // Just use this opportunity to invalidate local Cache Entry as well. + // Just use this opportunity to invalidate local Cache Entry as well. // If this silo is not the grain directory partition for this grain, it may have it in its cache. try { @@ -1188,7 +1188,7 @@ private async Task CallGrainDeactivateAndCleanupStreams(Activati if (TryGetActivationData(activation.ActivationId, out ignore) && activation.State == ActivationState.Deactivating) { - RequestContext.Clear(); // Clear any previous RC, so it does not leak into this call by mistake. + RequestContext.Clear(); // Clear any previous RC, so it does not leak into this call by mistake. await activation.Lifecycle.OnStop().WithCancellation(ct); } if (logger.IsEnabled(LogLevel.Debug)) logger.Debug(ErrorCode.Catalog_AfterCallingDeactivate, "Returned from calling {1} grain's OnDeactivateAsync() method {0}", activation, grainTypeName); @@ -1233,7 +1233,7 @@ private struct ActivationRegistrationResult /// public static readonly ActivationRegistrationResult Success = new ActivationRegistrationResult { - IsSuccess = true + IsSuccess = true }; public ActivationRegistrationResult(ActivationAddress existingActivationAddress) @@ -1242,7 +1242,7 @@ public ActivationRegistrationResult(ActivationAddress existingActivationAddress) ExistingActivationAddress = existingActivationAddress; IsSuccess = false; } - + /// /// Returns true if this instance represents a successful registration, false otherwise. /// @@ -1264,13 +1264,13 @@ private async Task RegisterActivationInGrainDirect { ActivationAddress address = activation.Address; - // Currently, the only grain type that is not registered in the Grain Directory is StatelessWorker. + // Currently, the only grain type that is not registered in the Grain Directory is StatelessWorker. // Among those that are registered in the directory, we currently do not have any multi activations. if (activation.IsUsingGrainDirectory) { var result = await scheduler.RunOrQueueTask(() => directory.RegisterAsync(address, singleActivation:true), this.SchedulingContext); if (address.Equals(result.Address)) return ActivationRegistrationResult.Success; - + return new ActivationRegistrationResult(existingActivationAddress: result.Address); } else if (activation.PlacedUsing is StatelessWorkerPlacement stPlacement) @@ -1302,7 +1302,7 @@ private async Task RegisterActivationInGrainDirect } } - // We currently don't have any other case for multiple activations except for StatelessWorker. + // We currently don't have any other case for multiple activations except for StatelessWorker. } /// @@ -1388,7 +1388,7 @@ private List TryGetActivationDatas(List addre } private void OnSiloStatusChange(SiloAddress updatedSilo, SiloStatus status) - { + { // ignore joining events and also events on myself. if (updatedSilo.Equals(LocalSilo)) return; diff --git a/src/Orleans.Runtime/Configuration/Options/SchedulingOptions.cs b/src/Orleans.Runtime/Configuration/Options/SchedulingOptions.cs index d84de246afc..01ec5909802 100644 --- a/src/Orleans.Runtime/Configuration/Options/SchedulingOptions.cs +++ b/src/Orleans.Runtime/Configuration/Options/SchedulingOptions.cs @@ -36,7 +36,7 @@ public class SchedulingOptions public static readonly TimeSpan DEFAULT_DELAY_WARNING_THRESHOLD = TimeSpan.FromMilliseconds(10000); // 10 seconds /// - /// ActivationSchedulingQuantum is a soft time limit on the duration of activation macro-turn (a number of micro-turns). + /// ActivationSchedulingQuantum is a soft time limit on the duration of activation macro-turn (a number of micro-turns). /// If an activation was running its micro-turns longer than this, we will give up the thread. /// If this is set to zero or a negative number, then the full work queue is drained (MaxWorkItemsPerTurn allowing). /// @@ -44,7 +44,7 @@ public class SchedulingOptions public static readonly TimeSpan DEFAULT_ACTIVATION_SCHEDULING_QUANTUM = TimeSpan.FromMilliseconds(100); /// - /// TurnWarningLengthThreshold is a soft time limit to generate trace warning when the micro-turn executes longer then this period in CPU. + /// TurnWarningLengthThreshold is a soft time limit to generate trace warning when the micro-turn executes longer then this period in CPU. /// public TimeSpan TurnWarningLengthThreshold { get; set; } = DEFAULT_TURN_WARNING_THRESHOLD; public static readonly TimeSpan DEFAULT_TURN_WARNING_THRESHOLD = TimeSpan.FromMilliseconds(200); diff --git a/src/Orleans.Runtime/Core/Dispatcher.cs b/src/Orleans.Runtime/Core/Dispatcher.cs index dfd5580cf49..358d232955e 100644 --- a/src/Orleans.Runtime/Core/Dispatcher.cs +++ b/src/Orleans.Runtime/Core/Dispatcher.cs @@ -33,8 +33,8 @@ internal class Dispatcher private readonly SchedulingOptions schedulingOptions; private readonly ILogger invokeWorkItemLogger; internal Dispatcher( - OrleansTaskScheduler scheduler, - ISiloMessageCenter transport, + OrleansTaskScheduler scheduler, + ISiloMessageCenter transport, Catalog catalog, IOptions messagingOptions, PlacementDirectorsManager placementDirectorsManager, @@ -93,10 +93,10 @@ public void ReceiveMessage(Message message) { Task ignore; ActivationData target = catalog.GetOrCreateActivation( - message.TargetAddress, + message.TargetAddress, message.IsNewPlacement, message.NewGrainType, - String.IsNullOrEmpty(message.GenericGrainType) ? null : message.GenericGrainType, + String.IsNullOrEmpty(message.GenericGrainType) ? null : message.GenericGrainType, message.RequestContextData, out ignore); @@ -125,7 +125,7 @@ public void ReceiveMessage(Message message) try { MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Non-existent activation"); - + var nea = ex as Catalog.NonExistentActivationException; if (nea == null) { @@ -199,9 +199,9 @@ await this.localGrainDirectory.UnregisterAfterNonexistingActivation( } public void RejectMessage( - Message message, - Message.RejectionTypes rejectType, - Exception exc, + Message message, + Message.RejectionTypes rejectType, + Exception exc, string rejectInfo = null) { if (message.Direction == Message.Directions.Request) @@ -276,13 +276,13 @@ private void ReceiveRequest(Message message, ActivationData targetActivation) { try { - CheckDeadlock(message); + CheckDeadlock(targetActivation, message); } catch (DeadlockException exc) { // Record that this message is no longer flowing through the system MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Deadlock"); - logger.Warn(ErrorCode.Dispatcher_DetectedDeadlock, + logger.Warn(ErrorCode.Dispatcher_DetectedDeadlock, "Detected Application Deadlock: {0}", exc.Message); // We want to send DeadlockException back as an application exception, rather than as a system rejection. SendResponse(message, Response.ExceptionResponse(exc)); @@ -316,18 +316,19 @@ private bool ActivationMayAcceptRequest(ActivationData targetActivation, Message } /// - /// Whether an incoming message can interleave + /// Whether an incoming message can interleave /// /// /// /// public bool CanInterleave(ActivationData targetActivation, Message incoming) { - bool canInterleave = + bool canInterleave = incoming.IsAlwaysInterleave || targetActivation.Running == null - || (targetActivation.Running.IsReadOnly && incoming.IsReadOnly) - || (schedulingOptions.AllowCallChainReentrancy && targetActivation.ActivationId.Equals(incoming.SendingActivation)) + || targetActivation.Running.IsReadOnly && incoming.IsReadOnly + || schedulingOptions.AllowCallChainReentrancy + && IsMessageACallChainLoop(incoming) || catalog.CanInterleave(targetActivation.ActivationId, incoming); return canInterleave; @@ -336,56 +337,78 @@ public bool CanInterleave(ActivationData targetActivation, Message incoming) /// /// https://github.com/dotnet/orleans/issues/3184 /// Checks whether reentrancy is allowed for calls to grains that are already part of the call chain. - /// Covers following case: grain A calls grain B, and while executing the invoked method B calls back to A. + /// Covers following case: grain A calls grain B, and while executing the invoked method B calls back to A. /// Design: Senders collection `RunningRequestsSenders` contains sending grains references - /// during duration of request processing. If target of outgoing request is found in that collection - + /// during duration of request processing. If target of outgoing request is found in that collection - /// such request will be marked as interleaving in order to prevent deadlocks. /// private void MarkSameCallChainMessageAsInterleaving(ActivationData sendingActivation, Message outgoing) { - if (!schedulingOptions.AllowCallChainReentrancy) - { - return; - } - - if (sendingActivation?.RunningRequestsSenders.Contains(outgoing.TargetActivation) == true) + if (schedulingOptions.AllowCallChainReentrancy && IsMessageACallChainLoop(outgoing)) { outgoing.IsAlwaysInterleave = true; } } /// - /// Check if the current message will cause deadlock. - /// Throw DeadlockException if yes. + /// Check if the current message will cause a loop in the existing call chain /// /// Message to analyze - private void CheckDeadlock(Message message) + /// + private bool IsMessageACallChainLoop(Message message) { var requestContext = message.RequestContextData; - object obj; + if (requestContext == null || - !requestContext.TryGetValue(RequestContext.CALL_CHAIN_REQUEST_CONTEXT_HEADER, out obj) || - obj == null) return; // first call in a chain + !requestContext.TryGetValue(RequestContext.CALL_CHAIN_REQUEST_CONTEXT_HEADER, out object obj) || + obj == null) return false; // first call in a chain var prevChain = ((IList)obj); ActivationId nextActivationId = message.TargetActivation; + // check if the target activation already appears in the call chain. foreach (object invocationObj in prevChain) { - var prevId = ((RequestInvocationHistory)invocationObj).ActivationId; - if (!prevId.Equals(nextActivationId) || catalog.CanInterleave(nextActivationId, message)) continue; + var prevId = ((RequestInvocationHistorySummary)invocationObj).ActivationId; + if (prevId.Equals(nextActivationId)) + { + return true; + } + } + + return false; + } + + /// + /// Check if the current message will cause deadlock. + /// Throw DeadlockException if yes. + /// + /// + /// Message to analyze + private void CheckDeadlock(ActivationData targetActivation, Message message) + { + if (IsMessageACallChainLoop(message) && !catalog.CanInterleave(targetActivation.ActivationId, message)) + { + IEnumerable> callChain = Enumerable.Empty>(); + string exceptionMessage = string.Empty; + var requestContext = message.RequestContextData; - var newChain = new List(); - newChain.AddRange(prevChain.Cast()); - newChain.Add(new RequestInvocationHistory(message.TargetGrain, message.TargetActivation, message.DebugContext)); + if (requestContext != null + && requestContext.TryGetValue(RequestContext.CALL_CHAIN_REQUEST_CONTEXT_HEADER, out object obj) + && obj is IList prevChain) + { + var newChain = new List(); + newChain.AddRange(prevChain.Cast()); + newChain.Add(new RequestInvocationHistory(message.TargetGrain, message.TargetActivation, + message.DebugContext)); + + exceptionMessage = Utils.EnumerableToString(newChain, + elem => $"{elem.GrainId}.{elem.DebugContext}"); + callChain = newChain.Select(req => new Tuple(req.GrainId, req.DebugContext)); + } throw new DeadlockException( - String.Format( - "Deadlock Exception for grain call chain {0}.", - Utils.EnumerableToString( - newChain, - elem => String.Format("{0}.{1}", elem.GrainId, elem.DebugContext))), - newChain.Select(req => new Tuple(req.GrainId, req.DebugContext)).ToList()); + $"Deadlock Exception for grain call chain {exceptionMessage}.", callChain.ToList()); } } @@ -464,13 +487,13 @@ private void EnqueueRequest(Message message, ActivationData targetActivation) } internal void ProcessRequestToInvalidActivation( - Message message, - ActivationAddress oldAddress, - ActivationAddress forwardingAddress, - string failedOperation, + Message message, + ActivationAddress oldAddress, + ActivationAddress forwardingAddress, + string failedOperation, Exception exc = null) { - // Just use this opportunity to invalidate local Cache Entry as well. + // Just use this opportunity to invalidate local Cache Entry as well. if (oldAddress != null) { this.localGrainDirectory.InvalidateCacheEntry(oldAddress); @@ -484,12 +507,12 @@ internal void ProcessRequestToInvalidActivation( internal void ProcessRequestsToInvalidActivation( List messages, ActivationAddress oldAddress, - ActivationAddress forwardingAddress, + ActivationAddress forwardingAddress, string failedOperation, Exception exc = null, bool rejectMessages = false) { - // Just use this opportunity to invalidate local Cache Entry as well. + // Just use this opportunity to invalidate local Cache Entry as well. if (oldAddress != null) { this.localGrainDirectory.InvalidateCacheEntry(oldAddress); @@ -515,7 +538,7 @@ internal void ProcessRequestsToInvalidActivation( { TryForwardRequest(message, oldAddress, forwardingAddress, failedOperation, exc); } - + } } ), catalog.SchedulingContext); @@ -632,7 +655,7 @@ private static bool MayForward(Message message, SiloMessagingOptions messagingOp /// - may buffer for transaction completion / commit if it ends a transaction /// - choose target placement address, maintaining send order /// - add ordering info and maintain send order - /// + /// /// /// /// @@ -778,7 +801,7 @@ internal void SendResponse(Message request, Response response) internal void SendSystemTargetMessage(Message message) { - message.Category = message.TargetGrain.Equals(Constants.MembershipOracleId) ? + message.Category = message.TargetGrain.Equals(Constants.MembershipOracleId) ? Message.Categories.Ping : Message.Categories.System; if (message.TargetSilo == null) @@ -809,7 +832,7 @@ public void TransportMessage(Message message, ActivationData sendingActivation = /// Invoked when an activation has finished a transaction and may be ready for additional transactions /// /// The activation that has just completed processing this message - /// The message that has just completed processing. + /// The message that has just completed processing. /// This will be null for the case of completion of Activate/Deactivate calls. internal void OnActivationCompletedRequest(ActivationData activation, Message message) { @@ -856,7 +879,7 @@ internal void RunMessagePump(ActivationData activation) var nextMessage = activation.PeekNextWaitingMessage(); if (nextMessage == null) continue; if (!ActivationMayAcceptRequest(activation, nextMessage)) continue; - + activation.DequeueNextWaitingMessage(); // we might be over-writing an already running read only request. HandleIncomingRequest(nextMessage, activation); diff --git a/src/Orleans.Runtime/Core/InsideRuntimeClient.cs b/src/Orleans.Runtime/Core/InsideRuntimeClient.cs index 7f94c24ba8e..e0a906e3380 100644 --- a/src/Orleans.Runtime/Core/InsideRuntimeClient.cs +++ b/src/Orleans.Runtime/Core/InsideRuntimeClient.cs @@ -99,7 +99,7 @@ public InsideRuntimeClient( private SiloAddress MySilo { get; } public GrainFactory ConcreteGrainFactory { get; } - + private Catalog Catalog => this.catalog ?? (this.catalog = this.ServiceProvider.GetRequiredService()); private ILocalGrainDirectory Directory @@ -252,12 +252,12 @@ public void SniffIncomingMessage(Message message) //// 1: //// Also record sending activation address for responses only in the cache. //// We don't record sending addresses for requests, since it is not clear that this silo ever wants to send messages to the grain sending this request. - //// However, it is sure that this silo does send messages to the sender of a reply. + //// However, it is sure that this silo does send messages to the sender of a reply. //// In most cases it will already have its address cached, unless it had a wrong outdated address cached and now this is a fresher address. //// It is anyway always safe to cache the replier address. - //// 2: + //// 2: //// after further thought decided not to do it. - //// It seems to better not bother caching the sender of a response at all, + //// It seems to better not bother caching the sender of a response at all, //// and instead to take a very occasional hit of a full remote look-up instead of this small but non-zero hit on every response. //if (message.Direction.Equals(Message.Directions.Response) && message.Result.Equals(Message.ResponseTypes.Success)) //{ @@ -289,9 +289,20 @@ public async Task Invoke(IAddressable target, IInvokable invokable, Message mess } RequestContextExtensions.Import(message.RequestContextData); - if (schedulingOptions.PerformDeadlockDetection && !message.TargetGrain.IsSystemTarget) + + if (!message.TargetGrain.IsSystemTarget) { - UpdateDeadlockInfoInRequestContext(new RequestInvocationHistory(message.TargetGrain, message.TargetActivation, message.DebugContext)); + if (schedulingOptions.PerformDeadlockDetection) + { + UpdateInvocationHistoryInRequestContext( + new RequestInvocationHistory(message.TargetGrain, message.TargetActivation, message.DebugContext)); + } + else if (schedulingOptions.AllowCallChainReentrancy) + { + UpdateInvocationHistoryInRequestContext( + new RequestInvocationHistorySummary(message.TargetActivation)); + } + // RequestContext is automatically saved in the msg upon send and propagated to the next hop // in RuntimeClient.CreateMessage -> RequestContextExtensions.ExportToMessage(message); } @@ -329,7 +340,7 @@ public async Task Invoke(IAddressable target, IInvokable invokable, Message mess !(target is IGrainExtension) && !TryInstallExtension(request.InterfaceId, invokable, message.GenericGrainType, ref invoker)) { - // We are trying the invoke a grain extension method on a grain + // We are trying the invoke a grain extension method on a grain // -- most likely reason is that the dynamic extension is not installed for this grain // So throw a specific exception here rather than a general InvalidCastException var error = String.Format( @@ -345,7 +356,7 @@ public async Task Invoke(IAddressable target, IInvokable invokable, Message mess throw exc; } - + var requestInvoker = new GrainMethodInvoker(target, request, invoker, GrainCallFilters, interfaceToImplementationMapping); await requestInvoker.Invoke(); resultObject = requestInvoker.Result; @@ -366,7 +377,7 @@ public async Task Invoke(IAddressable target, IInvokable invokable, Message mess if (transactionInfo != null) { transactionInfo.ReconcilePending(); - + // Record reason for abort, if not alread set transactionInfo.RecordException(exc1, serializationManager); @@ -508,7 +519,7 @@ private void SafeSendResponse(Message message, object resultObject) private static readonly Lazy> prepForRemotingLazy = new Lazy>(CreateExceptionPrepForRemotingMethod); - + private static Func CreateExceptionPrepForRemotingMethod() { var methodInfo = typeof(Exception).GetMethod( @@ -568,19 +579,20 @@ private void SafeSendExceptionResponse(Message message, Exception ex) } // assumes deadlock information was already loaded into RequestContext from the message - private static void UpdateDeadlockInfoInRequestContext(RequestInvocationHistory thisInvocation) + private static void UpdateInvocationHistoryInRequestContext(RequestInvocationHistorySummary thisInvocation) { - IList prevChain; - object obj = RequestContext.Get(RequestContext.CALL_CHAIN_REQUEST_CONTEXT_HEADER); - if (obj != null) + IList prevChain; + + if (RequestContext.Get(RequestContext.CALL_CHAIN_REQUEST_CONTEXT_HEADER) is IList obj) { - prevChain = ((IList)obj); + prevChain = obj; } else { - prevChain = new List(); + prevChain = new List(); RequestContext.Set(RequestContext.CALL_CHAIN_REQUEST_CONTEXT_HEADER, prevChain); } + // append this call to the end of the call chain. Update in place. prevChain.Add(thisInvocation); } @@ -612,7 +624,7 @@ public void ReceiveResponse(Message message) if (message.CacheInvalidationHeader == null) { // Remove from local directory cache. Note that SendingGrain is the original target, since message is the rejection response. - // If CacheMgmtHeader is present, we already did this. Otherwise, we left this code for backward compatability. + // If CacheMgmtHeader is present, we already did this. Otherwise, we left this code for backward compatability. // It should be retired as we move to use CacheMgmtHeader in all relevant places. this.Directory.InvalidateCacheEntry(message.SendingAddress); } @@ -635,7 +647,7 @@ public void ReceiveResponse(Message message) callbackData.TransactionInfo.Join(message.TransactionInfo); } // IMPORTANT: we do not schedule the response callback via the scheduler, since the only thing it does - // is to resolve/break the resolver. The continuations/waits that are based on this resolution will be scheduled as work items. + // is to resolve/break the resolver. The continuations/waits that are based on this resolution will be scheduled as work items. callbackData.DoCallback(message); } else diff --git a/test/Extensions/TesterAzureUtils/Streaming/HaloStreamSubscribeTests.cs b/test/Extensions/TesterAzureUtils/Streaming/HaloStreamSubscribeTests.cs index 7d359cd5d33..1791fb46795 100644 --- a/test/Extensions/TesterAzureUtils/Streaming/HaloStreamSubscribeTests.cs +++ b/test/Extensions/TesterAzureUtils/Streaming/HaloStreamSubscribeTests.cs @@ -75,12 +75,16 @@ public void Configure(ISiloHostBuilder hostBuilder) public override void Dispose() { base.Dispose(); - AzureQueueStreamProviderUtils.DeleteAllUsedAzureQueues(NullLoggerFactory.Instance, - AzureQueueUtilities.GenerateQueueNames(this.HostedCluster.Options.ClusterId, queueCount), - TestDefaultConfiguration.DataConnectionString).Wait(); - AzureQueueStreamProviderUtils.DeleteAllUsedAzureQueues(NullLoggerFactory.Instance, - AzureQueueUtilities.GenerateQueueNames($"{this.HostedCluster.Options.ClusterId}2", queueCount), - TestDefaultConfiguration.DataConnectionString).Wait(); + + if (HostedCluster != null) + { + AzureQueueStreamProviderUtils.DeleteAllUsedAzureQueues(NullLoggerFactory.Instance, + AzureQueueUtilities.GenerateQueueNames(this.HostedCluster.Options.ClusterId, queueCount), + TestDefaultConfiguration.DataConnectionString).Wait(); + AzureQueueStreamProviderUtils.DeleteAllUsedAzureQueues(NullLoggerFactory.Instance, + AzureQueueUtilities.GenerateQueueNames($"{this.HostedCluster.Options.ClusterId}2", queueCount), + TestDefaultConfiguration.DataConnectionString).Wait(); + } } } diff --git a/test/Grains/TestGrains/DeadlockGrain.cs b/test/Grains/TestGrains/DeadlockGrain.cs index 432b954e815..4b5fae35646 100644 --- a/test/Grains/TestGrains/DeadlockGrain.cs +++ b/test/Grains/TestGrains/DeadlockGrain.cs @@ -38,16 +38,18 @@ public class DeadlockNonReentrantGrain : Grain, IDeadlockNonReentrantGrain { private string Id { get { return String.Format("DeadlockNonReentrantGrain {0}", base.IdentityString); } } - public Task CallNext_1(List> callChain, int currCallIndex) + public async Task CallNext_1(List> callChain, int currCallIndex) { - this.GetLogger(Id).Info("Inside grain {0} CallNext_1().", Id); - return DeadlockGrain.CallNext(GrainFactory, callChain, currCallIndex); + this.GetLogger(Id).Info("Inside grain {0}.{1} CallNext_1().", Id, currCallIndex); + await DeadlockGrain.CallNext(GrainFactory, callChain, currCallIndex); + this.GetLogger(Id).Info("Inside grain {0}.{1} CallNext_1() Finished.", Id, currCallIndex); } - public Task CallNext_2(List> callChain, int currCallIndex) + public async Task CallNext_2(List> callChain, int currCallIndex) { - this.GetLogger(Id).Info("Inside grain {0} CallNext_2().", Id); - return DeadlockGrain.CallNext(GrainFactory, callChain, currCallIndex); + this.GetLogger(Id).Info("Inside grain {0}.{1} CallNext_2().", Id, currCallIndex); + await DeadlockGrain.CallNext(GrainFactory, callChain, currCallIndex); + this.GetLogger(Id).Info("Inside grain {0}.{1} CallNext_2() Finished.", Id, currCallIndex); } } diff --git a/test/TesterInternal/AllowCallChainReentrancyTests.cs b/test/TesterInternal/AllowCallChainReentrancyTests.cs new file mode 100644 index 00000000000..63d42c4fc67 --- /dev/null +++ b/test/TesterInternal/AllowCallChainReentrancyTests.cs @@ -0,0 +1,97 @@ +using System; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Orleans.Configuration; +using Orleans.Hosting; +using Orleans.TestingHost; +using TestExtensions; +using Xunit; +using Xunit.Abstractions; + +namespace UnitTests.General +{ + public class AllowCallChainReentrancyTests : OrleansTestingBase + { + private readonly Fixture fixture; + + public class Fixture : BaseTestClusterFixture + { + protected override void ConfigureTestCluster(TestClusterBuilder builder) + { + builder.AddSiloBuilderConfigurator(); + } + + private class SiloConfigurator : ISiloBuilderConfigurator + { + public void Configure(ISiloHostBuilder hostBuilder) + { + hostBuilder.Configure(options => + { + options.PerformDeadlockDetection = false; + options.AllowCallChainReentrancy = true; + }); + + } + } + } + + private const int numIterations = 30; + + private readonly CallChainReentrancyTestHelper testHelper; + + public AllowCallChainReentrancyTests(ITestOutputHelper output) + { + if(output == null) throw new ArgumentNullException(nameof(output)); + + this.fixture = new Fixture(); + testHelper = new CallChainReentrancyTestHelper + { + Random = random, + Fixture = fixture, + NumIterations = numIterations + }; + } + + // 1) Allowed reentrancy A, A + [Fact, TestCategory("Functional"), TestCategory("Deadlock")] + public async Task DeadlockDetection_1() + { + await testHelper.DeadlockDetection_1(); + } + + // 2) Allowed reentrancy on non-reentrant grains A, B, A + [Fact, TestCategory("Functional"), TestCategory("Deadlock")] + public async Task DeadlockDetection_2() + { + await testHelper.DeadlockDetection_2(); + } + + // 3) Allowed reentrancy X, A, X, A + [Fact, TestCategory("Functional"), TestCategory("Deadlock")] + public async Task DeadlockDetection_3() + { + await testHelper.DeadlockDetection_3(); + } + + // 4) No Deadlock X, X + [Fact, TestCategory("Functional"), TestCategory("Deadlock")] + public async Task DeadlockDetection_4() + { + await testHelper.DeadlockDetection_4(); + } + + // 5) No Deadlock X, A, X + [Fact, TestCategory("Functional"), TestCategory("Deadlock")] + public async Task DeadlockDetection_5() + { + await testHelper.DeadlockDetection_5(); + } + + // 6) Allowed reentrancy on non-reentrant grains A, B, C, A + [Fact, TestCategory("Functional"), TestCategory("Deadlock")] + public async Task DeadlockDetection_6() + { + await testHelper.DeadlockDetection_6(); + } + } +} \ No newline at end of file diff --git a/test/TesterInternal/DeadlockDetectionWithAllowCallChainReentrancyTests.cs b/test/TesterInternal/CallChainReentrancyTestHelper.cs similarity index 50% rename from test/TesterInternal/DeadlockDetectionWithAllowCallChainReentrancyTests.cs rename to test/TesterInternal/CallChainReentrancyTestHelper.cs index 0b348d0eb9a..46a10905331 100644 --- a/test/TesterInternal/DeadlockDetectionWithAllowCallChainReentrancyTests.cs +++ b/test/TesterInternal/CallChainReentrancyTestHelper.cs @@ -1,66 +1,35 @@ using System; using System.Collections.Generic; using System.Threading.Tasks; -using Orleans; -using Orleans.TestingHost; -using UnitTests.GrainInterfaces; -using Xunit; using TestExtensions; -using Orleans.Hosting; -using Orleans.Configuration; +using UnitTests.GrainInterfaces; namespace UnitTests.General { - public class DeadlockDetectionWithAllowCallChainReentrancyTests : OrleansTestingBase, IClassFixture + public class CallChainReentrancyTestHelper { - private readonly Fixture fixture; - - public class Fixture : BaseTestClusterFixture - { - protected override void ConfigureTestCluster(TestClusterBuilder builder) - { - builder.AddSiloBuilderConfigurator(); - } - - private class SiloConfigurator : ISiloBuilderConfigurator - { - public void Configure(ISiloHostBuilder hostBuilder) - { - hostBuilder.Configure(options => - { - options.PerformDeadlockDetection = true; - options.AllowCallChainReentrancy = true; - }); - } - } - - } - - private const int numIterations = 30; - - public DeadlockDetectionWithAllowCallChainReentrancyTests(Fixture fixture) - { - this.fixture = fixture; - } + public Random Random { get; set; } + public BaseTestClusterFixture Fixture { get; set; } + public int NumIterations { get; set; } // 2 silos, loop across all cases (to force all grains to be local and remote): - // Non Reentrant A, B - // Reentrant C + // Non Reentrant A, B, C + // Reentrant X // 1) No Deadlock A, A // 2) No Deadlock A, B, A - // 3) No Deadlock C, A, C, A - // 4) No Deadlock C, C - // 5) No Deadlock C, A, C + // 3) No Deadlock X, A, X, A + // 4) No Deadlock X, X + // 5) No Deadlock X, A, X + // 6) No Deadlock A, B, C, A // 1) Allowed reentrancy A, A - [Fact, TestCategory("Functional"), TestCategory("Deadlock")] public async Task DeadlockDetection_1() { - long baseGrainId = random.Next(); - for (int i = 0; i < numIterations; i++) + long baseGrainId = this.Random.Next(); + for (int i = 0; i < this.NumIterations; i++) { long grainId = baseGrainId + i; - IDeadlockNonReentrantGrain firstGrain = this.fixture.GrainFactory.GetGrain(grainId); + IDeadlockNonReentrantGrain firstGrain = this.Fixture.GrainFactory.GetGrain(grainId); List> callChain = new List>(); callChain.Add(new Tuple(grainId, true)); callChain.Add(new Tuple(grainId, true)); @@ -69,15 +38,14 @@ public async Task DeadlockDetection_1() } // 2) Allowed reentrancy on non-reentrant grains A, B, A - [Fact, TestCategory("Functional"), TestCategory("Deadlock")] public async Task DeadlockDetection_2() { - long baseGrainId = random.Next(); + long baseGrainId = this.Random.Next(); long bBase = 100; - for (int i = 0; i < numIterations; i++) + for (int i = 0; i < this.NumIterations; i++) { long grainId = baseGrainId + i; - IDeadlockNonReentrantGrain firstGrain = this.fixture.GrainFactory.GetGrain(grainId); + IDeadlockNonReentrantGrain firstGrain = this.Fixture.GrainFactory.GetGrain(grainId); List> callChain = new List>(); callChain.Add(new Tuple(grainId, true)); callChain.Add(new Tuple(bBase + grainId, true)); @@ -86,60 +54,76 @@ public async Task DeadlockDetection_2() } } - // 3) Allowed reentrancy C, A, C, A - [Fact, TestCategory("Functional"), TestCategory("Deadlock")] + // 3) Allowed reentrancy X, A, X, A public async Task DeadlockDetection_3() { - long baseGrainId = random.Next(); - long cBase = 200; - for (int i = 0; i < numIterations; i++) + long baseGrainId = this.Random.Next(); + long xBase = 1000; + for (int i = 0; i < this.NumIterations; i++) { long grainId = baseGrainId + i; - IDeadlockReentrantGrain firstGrain = this.fixture.GrainFactory.GetGrain(grainId); + IDeadlockReentrantGrain firstGrain = this.Fixture.GrainFactory.GetGrain(grainId); List> callChain = new List>(); - callChain.Add(new Tuple(cBase + grainId, false)); + callChain.Add(new Tuple(xBase + grainId, false)); callChain.Add(new Tuple(grainId, true)); - callChain.Add(new Tuple(cBase + grainId, false)); + callChain.Add(new Tuple(xBase + grainId, false)); callChain.Add(new Tuple(grainId, true)); await firstGrain.CallNext_1(callChain, 1); } } - // 4) No Deadlock C, C - [Fact, TestCategory("Functional"), TestCategory("Deadlock")] + // 4) No Deadlock X, X public async Task DeadlockDetection_4() { - long baseGrainId = random.Next(); - long cBase = 200; - for (int i = 0; i < numIterations; i++) + long baseGrainId = this.Random.Next(); + long xBase = 1000; + for (int i = 0; i < this.NumIterations; i++) { long grainId = baseGrainId + i; - IDeadlockReentrantGrain firstGrain = this.fixture.GrainFactory.GetGrain(grainId); + IDeadlockReentrantGrain firstGrain = this.Fixture.GrainFactory.GetGrain(grainId); List> callChain = new List>(); - callChain.Add(new Tuple(cBase + grainId, false)); - callChain.Add(new Tuple(cBase + grainId, false)); + callChain.Add(new Tuple(xBase + grainId, false)); + callChain.Add(new Tuple(xBase + grainId, false)); await firstGrain.CallNext_1(callChain, 1); } } - // 5) No Deadlock C, A, C - [Fact, TestCategory("Functional"), TestCategory("Deadlock")] + // 5) No Deadlock X, A, X public async Task DeadlockDetection_5() { - long baseGrainId = random.Next(); - long cBase = 200; - for (int i = 0; i < numIterations; i++) + long baseGrainId = this.Random.Next(); + long xBase = 1000; + for (int i = 0; i < this.NumIterations; i++) { long grainId = baseGrainId + i; - IDeadlockReentrantGrain firstGrain = this.fixture.GrainFactory.GetGrain(grainId); + IDeadlockReentrantGrain firstGrain = this.Fixture.GrainFactory.GetGrain(grainId); List> callChain = new List>(); - callChain.Add(new Tuple(cBase + grainId, false)); + callChain.Add(new Tuple(xBase + grainId, false)); callChain.Add(new Tuple(grainId, true)); - callChain.Add(new Tuple(cBase + grainId, false)); + callChain.Add(new Tuple(xBase + grainId, false)); + + await firstGrain.CallNext_1(callChain, 1); + } + } + // 6) Allowed reentrancy on non-reentrant grains only when using full chain reentrancy A, B, C, A + public async Task DeadlockDetection_6() + { + long baseGrainId = this.Random.Next(); + long bBase = 100; + long cBase = 200; + for (int i = 0; i < this.NumIterations; i++) + { + long grainId = baseGrainId + i; + IDeadlockNonReentrantGrain firstGrain = this.Fixture.GrainFactory.GetGrain(grainId); + List> callChain = new List>(); + callChain.Add(new Tuple(grainId, true)); + callChain.Add(new Tuple(bBase + grainId, true)); + callChain.Add(new Tuple(cBase + grainId, true)); + callChain.Add(new Tuple(grainId, true)); await firstGrain.CallNext_1(callChain, 1); } } } -} +} \ No newline at end of file diff --git a/test/TesterInternal/DeadlockDetectionWithAllowCallChainSingleCallReentrancyTests.cs b/test/TesterInternal/DeadlockDetectionWithAllowCallChainSingleCallReentrancyTests.cs new file mode 100644 index 00000000000..3242967e81f --- /dev/null +++ b/test/TesterInternal/DeadlockDetectionWithAllowCallChainSingleCallReentrancyTests.cs @@ -0,0 +1,97 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Orleans; +using Orleans.TestingHost; +using UnitTests.GrainInterfaces; +using Xunit; +using TestExtensions; +using Orleans.Hosting; +using Orleans.Configuration; + +namespace UnitTests.General +{ + public class DeadlockDetectionWithAllowCallChainSingleCallReentrancyTests : OrleansTestingBase, IClassFixture + { + private readonly Fixture fixture; + + public class Fixture : BaseTestClusterFixture + { + protected override void ConfigureTestCluster(TestClusterBuilder builder) + { + builder.AddSiloBuilderConfigurator(); + } + + private class SiloConfigurator : ISiloBuilderConfigurator + { + public void Configure(ISiloHostBuilder hostBuilder) + { + hostBuilder.Configure(options => + { + options.PerformDeadlockDetection = true; + options.AllowCallChainReentrancy = true; + }); + } + } + + } + + private const int numIterations = 30; + + private readonly CallChainReentrancyTestHelper testHelper; + + public DeadlockDetectionWithAllowCallChainSingleCallReentrancyTests(Fixture fixture) + { + this.fixture = fixture; + testHelper = new CallChainReentrancyTestHelper() + { + Random = random, + Fixture = fixture, + NumIterations = numIterations + }; + } + + + // 1) Allowed reentrancy A, A + [Fact, TestCategory("Functional"), TestCategory("Deadlock")] + public async Task DeadlockDetection_1() + { + await testHelper.DeadlockDetection_1(); + } + + // 2) Allowed reentrancy on non-reentrant grains A, B, A + [Fact, TestCategory("Functional"), TestCategory("Deadlock")] + public async Task DeadlockDetection_2() + { + await testHelper.DeadlockDetection_2(); + } + + // 3) Allowed reentrancy X, A, X, A + [Fact, TestCategory("Functional"), TestCategory("Deadlock")] + public async Task DeadlockDetection_3() + { + await testHelper.DeadlockDetection_3(); + } + + // 4) No Deadlock X, X + [Fact, TestCategory("Functional"), TestCategory("Deadlock")] + public async Task DeadlockDetection_4() + { + await testHelper.DeadlockDetection_4(); + } + + // 5) No Deadlock X, A, X + [Fact, TestCategory("Functional"), TestCategory("Deadlock")] + public async Task DeadlockDetection_5() + { + await testHelper.DeadlockDetection_5(); + } + + // 6) Allowed reentrancy on non-reentrant grains A, B, C, A + [Fact, TestCategory("Functional"), TestCategory("Deadlock")] + public async Task DeadlockDetection_6() + { + await testHelper.DeadlockDetection_6(); + } + } +} diff --git a/test/TesterInternal/DeadlockDetectionWithoutAllowCallChainReentrancyTests.cs b/test/TesterInternal/DeadlockDetectionWithoutAllowCallChainReentrancyTests.cs index ca4a9744ee5..3a92bbd4035 100644 --- a/test/TesterInternal/DeadlockDetectionWithoutAllowCallChainReentrancyTests.cs +++ b/test/TesterInternal/DeadlockDetectionWithoutAllowCallChainReentrancyTests.cs @@ -44,6 +44,8 @@ public DeadlockDetectionWithoutAllowCallChainReentrancyTests(Fixture fixture) this.fixture = fixture; } + // TODO add test 6 + // 2 silos, loop across all cases (to force all grains to be local and remote): // Non Reentrant A, B // Reentrant C diff --git a/test/TesterInternal/TestLoggingProvider.cs b/test/TesterInternal/TestLoggingProvider.cs new file mode 100644 index 00000000000..1410410da0b --- /dev/null +++ b/test/TesterInternal/TestLoggingProvider.cs @@ -0,0 +1,46 @@ +using System; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions.Internal; + +namespace UnitTests.General +{ + public class TestLoggingProvider : ILoggerProvider + { + private readonly TestLogger logger; + + public TestLoggingProvider(Action logMethod) + { + this.logger = new TestLogger(logMethod); + } + + private class TestLogger : ILogger + { + private readonly Action logMethod; + + public TestLogger(Action logMethod) + { + this.logMethod = logMethod; + } + + void ILogger.Log(LogLevel logLevel, EventId eventId, TState state, Exception exception, + Func formatter) + { + this.logMethod($"{logLevel}: ({eventId}-{eventId})-{formatter(state, exception)}"); + } + + bool ILogger.IsEnabled(LogLevel logLevel) => true; + + IDisposable ILogger.BeginScope(TState state) => NullScope.Instance; + + } + + public void Dispose() + { + } + + public ILogger CreateLogger(string categoryName) + { + return this.logger; + } + } +} \ No newline at end of file