From ec466627194c1e342a4980c835615567b98f999b Mon Sep 17 00:00:00 2001 From: Andrew Hesky Date: Sun, 21 Oct 2018 11:28:08 -0700 Subject: [PATCH 01/10] Add failing test for reentrancy --- ...ectionWithAllowCallChainReentrancyTests.cs | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/test/TesterInternal/DeadlockDetectionWithAllowCallChainReentrancyTests.cs b/test/TesterInternal/DeadlockDetectionWithAllowCallChainReentrancyTests.cs index 0b348d0eb9a..95f6732e12f 100644 --- a/test/TesterInternal/DeadlockDetectionWithAllowCallChainReentrancyTests.cs +++ b/test/TesterInternal/DeadlockDetectionWithAllowCallChainReentrancyTests.cs @@ -44,13 +44,14 @@ public DeadlockDetectionWithAllowCallChainReentrancyTests(Fixture fixture) } // 2 silos, loop across all cases (to force all grains to be local and remote): - // Non Reentrant A, B + // Non Reentrant A, B, D // Reentrant C // 1) No Deadlock A, A // 2) No Deadlock A, B, A // 3) No Deadlock C, A, C, A // 4) No Deadlock C, C // 5) No Deadlock C, A, C + // 6) No Deadlock A, B, D, A // 1) Allowed reentrancy A, A [Fact, TestCategory("Functional"), TestCategory("Deadlock")] @@ -141,5 +142,25 @@ public async Task DeadlockDetection_5() await firstGrain.CallNext_1(callChain, 1); } } + + // 6) Allowed reentrancy on non-reentrant grains A, B, D, A + [Fact, TestCategory("Functional"), TestCategory("Deadlock")] + public async Task DeadlockDetection_6() + { + long baseGrainId = random.Next(); + long bBase = 100; + long dBase = 200; + for (int i = 0; i < numIterations; i++) + { + long grainId = baseGrainId + i; + IDeadlockNonReentrantGrain firstGrain = this.fixture.GrainFactory.GetGrain(grainId); + List> callChain = new List>(); + callChain.Add(new Tuple(grainId, true)); + callChain.Add(new Tuple(bBase + grainId, true)); + callChain.Add(new Tuple(dBase + grainId, true)); + callChain.Add(new Tuple(grainId, true)); + await firstGrain.CallNext_1(callChain, 1); + } + } } } From dbbaeee8a705a099423e5e1d6f57573518183a02 Mon Sep 17 00:00:00 2001 From: Andrew Hesky Date: Sun, 21 Oct 2018 18:44:48 -0700 Subject: [PATCH 02/10] Code in place, tests failing --- .../OptionLogger/DefaultOptionsFormatter.cs | 3 +- .../Messaging/RequestInvocationHistory.cs | 24 ++- .../LegacyClusterConfigurationExtensions.cs | 9 +- .../Options/SchedulingOptions.cs | 30 +++- src/Orleans.Runtime/Core/Dispatcher.cs | 129 ++++++++----- .../Core/InsideRuntimeClient.cs | 43 +++-- ...llowCallChainReentrancyEntireChainTests.cs | 169 ++++++++++++++++++ ...llowCallChainSingleCallReentrancyTests.cs} | 48 ++--- ...ionWithoutAllowCallChainReentrancyTests.cs | 4 +- 9 files changed, 357 insertions(+), 102 deletions(-) create mode 100644 test/TesterInternal/AllowCallChainReentrancyEntireChainTests.cs rename test/TesterInternal/{DeadlockDetectionWithAllowCallChainReentrancyTests.cs => DeadlockDetectionWithAllowCallChainSingleCallReentrancyTests.cs} (82%) diff --git a/src/Orleans.Core/Configuration/OptionLogger/DefaultOptionsFormatter.cs b/src/Orleans.Core/Configuration/OptionLogger/DefaultOptionsFormatter.cs index 456360a173f..31cd74adaef 100644 --- a/src/Orleans.Core/Configuration/OptionLogger/DefaultOptionsFormatter.cs +++ b/src/Orleans.Core/Configuration/OptionLogger/DefaultOptionsFormatter.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Linq; using System.Reflection; +using System.Security.Cryptography.X509Certificates; namespace Orleans { @@ -87,7 +88,7 @@ private IEnumerable FormatProperty(PropertyInfo property) } } - internal class DefaultOptionsFormatterResolver : IOptionFormatterResolver + internal class DefaultOptionsFormatterResolver : IOptionFormatterResolver where T: class, new() { private IOptionsSnapshot optionsSnapshot; diff --git a/src/Orleans.Core/Messaging/RequestInvocationHistory.cs b/src/Orleans.Core/Messaging/RequestInvocationHistory.cs index 90ad9b82450..c345a2cc3ab 100644 --- a/src/Orleans.Core/Messaging/RequestInvocationHistory.cs +++ b/src/Orleans.Core/Messaging/RequestInvocationHistory.cs @@ -4,22 +4,36 @@ namespace Orleans.Runtime { // used for tracking request invocation history for deadlock detection. [Serializable] - internal sealed class RequestInvocationHistory + internal sealed class RequestInvocationHistory : RequestInvocationHistorySummary { public GrainId GrainId { get; private set; } - public ActivationId ActivationId { get; private set; } public string DebugContext { get; private set; } - public RequestInvocationHistory(GrainId grainId, ActivationId activationId, string debugContext) + public RequestInvocationHistory(GrainId grainId, ActivationId activationId, string debugContext) : base(activationId) { this.GrainId = grainId; - this.ActivationId = activationId; DebugContext = debugContext; } public override string ToString() { - return String.Format("RequestInvocationHistory {0}:{1}:{2}", GrainId, ActivationId, DebugContext); + return $"RequestInvocationHistory {GrainId}:{ActivationId}:{DebugContext}"; + } + } + // used for tracking request invocation history for call chain reentrancy + [Serializable] + internal class RequestInvocationHistorySummary + { + public ActivationId ActivationId { get; private set; } + + public RequestInvocationHistorySummary(ActivationId activationId) + { + this.ActivationId = activationId; + } + + public override string ToString() + { + return $"RequestInvocationHistorySummary {ActivationId}"; } } } diff --git a/src/Orleans.Runtime.Legacy/Configuration/LegacyClusterConfigurationExtensions.cs b/src/Orleans.Runtime.Legacy/Configuration/LegacyClusterConfigurationExtensions.cs index 02d281c3bd1..96656c00360 100644 --- a/src/Orleans.Runtime.Legacy/Configuration/LegacyClusterConfigurationExtensions.cs +++ b/src/Orleans.Runtime.Legacy/Configuration/LegacyClusterConfigurationExtensions.cs @@ -185,11 +185,14 @@ private static void AddLegacyClusterConfigurationSupport(IServiceCollection serv var nodeConfig = configuration.GetOrCreateNodeConfigurationForSilo(siloOptions.Value.SiloName); options.ExcludedGrainTypes.AddRange(nodeConfig.ExcludedGrainTypes); }); - + services.AddOptions() .Configure((options, config) => { - options.AllowCallChainReentrancy = config.AllowCallChainReentrancy; + options.CallChainReentrancy = config.AllowCallChainReentrancy ? + SchedulingOptions.CallChainReentrancyStrategy.SingleCall + : SchedulingOptions.CallChainReentrancyStrategy.None; + options.PerformDeadlockDetection = config.PerformDeadlockDetection; }) .Configure((options, nodeConfig) => @@ -273,7 +276,7 @@ private static void AddLegacyClusterConfigurationSupport(IServiceCollection serv { options.IsRunningAsUnitTest = config.IsRunningAsUnitTest; }); - + services.AddOptions() .Configure((options, config) => { diff --git a/src/Orleans.Runtime/Configuration/Options/SchedulingOptions.cs b/src/Orleans.Runtime/Configuration/Options/SchedulingOptions.cs index d84de246afc..e0d4749f377 100644 --- a/src/Orleans.Runtime/Configuration/Options/SchedulingOptions.cs +++ b/src/Orleans.Runtime/Configuration/Options/SchedulingOptions.cs @@ -9,6 +9,19 @@ namespace Orleans.Configuration /// public class SchedulingOptions { + /// + /// Configuration type that controls the type of the grain directory caching algorithm that silo use. + /// + public enum CallChainReentrancyStrategy + { + /// Don't allow reentrancy. + None, + /// Allow reentrancy for a single call (A->B->A). + SingleCall, + /// Allow reentrancy for for entire call chain (A->B->C->A). + EntireChain + } + /// /// Whether or not to perform deadlock detection. /// @@ -18,8 +31,17 @@ public class SchedulingOptions /// /// Whether or not to allow reentrancy for calls within the same call chain. /// - public bool AllowCallChainReentrancy { get; set; } = DEFAULT_ALLOW_CALL_CHAIN_REENTRANCY; - public const bool DEFAULT_ALLOW_CALL_CHAIN_REENTRANCY = true; + [Obsolete("AllowCallChainReentrancy is obsolete, use CallChainReentrancy. None is equivalent to false, SingleCall is equivalent to true")] + public bool AllowCallChainReentrancy { + get => CallChainReentrancy != CallChainReentrancyStrategy.None; + set => CallChainReentrancy = value ? CallChainReentrancyStrategy.SingleCall : CallChainReentrancyStrategy.None; + } + + /// + /// How to handle reentrancy for calls within the same call chain. + /// + public CallChainReentrancyStrategy CallChainReentrancy { get; set; } = DEFAULT_CALL_CHAIN_REENTRANCY; + public const CallChainReentrancyStrategy DEFAULT_CALL_CHAIN_REENTRANCY = CallChainReentrancyStrategy.SingleCall; /// /// The MaxActiveThreads attribute specifies the maximum number of simultaneous active threads the scheduler will allow. @@ -36,7 +58,7 @@ public class SchedulingOptions public static readonly TimeSpan DEFAULT_DELAY_WARNING_THRESHOLD = TimeSpan.FromMilliseconds(10000); // 10 seconds /// - /// ActivationSchedulingQuantum is a soft time limit on the duration of activation macro-turn (a number of micro-turns). + /// ActivationSchedulingQuantum is a soft time limit on the duration of activation macro-turn (a number of micro-turns). /// If an activation was running its micro-turns longer than this, we will give up the thread. /// If this is set to zero or a negative number, then the full work queue is drained (MaxWorkItemsPerTurn allowing). /// @@ -44,7 +66,7 @@ public class SchedulingOptions public static readonly TimeSpan DEFAULT_ACTIVATION_SCHEDULING_QUANTUM = TimeSpan.FromMilliseconds(100); /// - /// TurnWarningLengthThreshold is a soft time limit to generate trace warning when the micro-turn executes longer then this period in CPU. + /// TurnWarningLengthThreshold is a soft time limit to generate trace warning when the micro-turn executes longer then this period in CPU. /// public TimeSpan TurnWarningLengthThreshold { get; set; } = DEFAULT_TURN_WARNING_THRESHOLD; public static readonly TimeSpan DEFAULT_TURN_WARNING_THRESHOLD = TimeSpan.FromMilliseconds(200); diff --git a/src/Orleans.Runtime/Core/Dispatcher.cs b/src/Orleans.Runtime/Core/Dispatcher.cs index dfd5580cf49..259e136aab5 100644 --- a/src/Orleans.Runtime/Core/Dispatcher.cs +++ b/src/Orleans.Runtime/Core/Dispatcher.cs @@ -33,8 +33,8 @@ internal class Dispatcher private readonly SchedulingOptions schedulingOptions; private readonly ILogger invokeWorkItemLogger; internal Dispatcher( - OrleansTaskScheduler scheduler, - ISiloMessageCenter transport, + OrleansTaskScheduler scheduler, + ISiloMessageCenter transport, Catalog catalog, IOptions messagingOptions, PlacementDirectorsManager placementDirectorsManager, @@ -93,10 +93,10 @@ public void ReceiveMessage(Message message) { Task ignore; ActivationData target = catalog.GetOrCreateActivation( - message.TargetAddress, + message.TargetAddress, message.IsNewPlacement, message.NewGrainType, - String.IsNullOrEmpty(message.GenericGrainType) ? null : message.GenericGrainType, + String.IsNullOrEmpty(message.GenericGrainType) ? null : message.GenericGrainType, message.RequestContextData, out ignore); @@ -125,7 +125,7 @@ public void ReceiveMessage(Message message) try { MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Non-existent activation"); - + var nea = ex as Catalog.NonExistentActivationException; if (nea == null) { @@ -199,9 +199,9 @@ await this.localGrainDirectory.UnregisterAfterNonexistingActivation( } public void RejectMessage( - Message message, - Message.RejectionTypes rejectType, - Exception exc, + Message message, + Message.RejectionTypes rejectType, + Exception exc, string rejectInfo = null) { if (message.Direction == Message.Directions.Request) @@ -282,7 +282,7 @@ private void ReceiveRequest(Message message, ActivationData targetActivation) { // Record that this message is no longer flowing through the system MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Deadlock"); - logger.Warn(ErrorCode.Dispatcher_DetectedDeadlock, + logger.Warn(ErrorCode.Dispatcher_DetectedDeadlock, "Detected Application Deadlock: {0}", exc.Message); // We want to send DeadlockException back as an application exception, rather than as a system rejection. SendResponse(message, Response.ExceptionResponse(exc)); @@ -316,18 +316,21 @@ private bool ActivationMayAcceptRequest(ActivationData targetActivation, Message } /// - /// Whether an incoming message can interleave + /// Whether an incoming message can interleave /// /// /// /// public bool CanInterleave(ActivationData targetActivation, Message incoming) { - bool canInterleave = + bool canInterleave = incoming.IsAlwaysInterleave || targetActivation.Running == null - || (targetActivation.Running.IsReadOnly && incoming.IsReadOnly) - || (schedulingOptions.AllowCallChainReentrancy && targetActivation.ActivationId.Equals(incoming.SendingActivation)) + || targetActivation.Running.IsReadOnly && incoming.IsReadOnly + || schedulingOptions.CallChainReentrancy == SchedulingOptions.CallChainReentrancyStrategy.SingleCall + && targetActivation.ActivationId.Equals(incoming.SendingActivation) // TODO, consider using the same method for both modes + || schedulingOptions.CallChainReentrancy == SchedulingOptions.CallChainReentrancyStrategy.EntireChain + && IsMessageACallChainLoop(incoming) || catalog.CanInterleave(targetActivation.ActivationId, incoming); return canInterleave; @@ -336,56 +339,90 @@ public bool CanInterleave(ActivationData targetActivation, Message incoming) /// /// https://github.com/dotnet/orleans/issues/3184 /// Checks whether reentrancy is allowed for calls to grains that are already part of the call chain. - /// Covers following case: grain A calls grain B, and while executing the invoked method B calls back to A. + /// Covers following case: grain A calls grain B, and while executing the invoked method B calls back to A. /// Design: Senders collection `RunningRequestsSenders` contains sending grains references - /// during duration of request processing. If target of outgoing request is found in that collection - + /// during duration of request processing. If target of outgoing request is found in that collection - /// such request will be marked as interleaving in order to prevent deadlocks. /// private void MarkSameCallChainMessageAsInterleaving(ActivationData sendingActivation, Message outgoing) { - if (!schedulingOptions.AllowCallChainReentrancy) + if (schedulingOptions.CallChainReentrancy == SchedulingOptions.CallChainReentrancyStrategy.None) { return; } - if (sendingActivation?.RunningRequestsSenders.Contains(outgoing.TargetActivation) == true) + // TODO consider using the same method for both strategies + if (schedulingOptions.CallChainReentrancy == SchedulingOptions.CallChainReentrancyStrategy.SingleCall + && sendingActivation?.RunningRequestsSenders.Contains(outgoing.TargetActivation) == true) + { + outgoing.IsAlwaysInterleave = true; + } + + if (schedulingOptions.CallChainReentrancy == SchedulingOptions.CallChainReentrancyStrategy.EntireChain + && IsMessageACallChainLoop(outgoing)) { outgoing.IsAlwaysInterleave = true; } } /// - /// Check if the current message will cause deadlock. - /// Throw DeadlockException if yes. + /// Check if the current message will cause a loop in the existing call chain /// /// Message to analyze - private void CheckDeadlock(Message message) + /// + private bool IsMessageACallChainLoop(Message message) { var requestContext = message.RequestContextData; - object obj; + if (requestContext == null || - !requestContext.TryGetValue(RequestContext.CALL_CHAIN_REQUEST_CONTEXT_HEADER, out obj) || - obj == null) return; // first call in a chain + !requestContext.TryGetValue(RequestContext.CALL_CHAIN_REQUEST_CONTEXT_HEADER, out object obj) || + obj == null) return false; // first call in a chain var prevChain = ((IList)obj); ActivationId nextActivationId = message.TargetActivation; // check if the target activation already appears in the call chain. foreach (object invocationObj in prevChain) { - var prevId = ((RequestInvocationHistory)invocationObj).ActivationId; - if (!prevId.Equals(nextActivationId) || catalog.CanInterleave(nextActivationId, message)) continue; + var prevId = ((RequestInvocationHistorySummary)invocationObj).ActivationId; + if (prevId.Equals(nextActivationId) && !catalog.CanInterleave(nextActivationId, message)) + { + return true; + } + } + + return false; + } + + /// + /// Check if the current message will cause deadlock. + /// Throw DeadlockException if yes. + /// + /// Message to analyze + private void CheckDeadlock(Message message) + { + if (IsMessageACallChainLoop(message) + && schedulingOptions.CallChainReentrancy != SchedulingOptions.CallChainReentrancyStrategy.EntireChain) + { + IEnumerable> callChain = Enumerable.Empty>(); + string exceptionMessage = string.Empty; + var requestContext = message.RequestContextData; - var newChain = new List(); - newChain.AddRange(prevChain.Cast()); - newChain.Add(new RequestInvocationHistory(message.TargetGrain, message.TargetActivation, message.DebugContext)); + if (requestContext != null + && requestContext.TryGetValue(RequestContext.CALL_CHAIN_REQUEST_CONTEXT_HEADER, out object obj) + && obj is IList prevChain) + { + var newChain = new List(); + newChain.AddRange(prevChain.Cast()); + newChain.Add(new RequestInvocationHistory(message.TargetGrain, message.TargetActivation, + message.DebugContext)); + + exceptionMessage = Utils.EnumerableToString(newChain, + elem => $"{elem.GrainId}.{elem.DebugContext}"); + callChain = newChain.Select(req => new Tuple(req.GrainId, req.DebugContext)); + } throw new DeadlockException( - String.Format( - "Deadlock Exception for grain call chain {0}.", - Utils.EnumerableToString( - newChain, - elem => String.Format("{0}.{1}", elem.GrainId, elem.DebugContext))), - newChain.Select(req => new Tuple(req.GrainId, req.DebugContext)).ToList()); + $"Deadlock Exception for grain call chain {exceptionMessage}.", callChain.ToList()); } } @@ -464,13 +501,13 @@ private void EnqueueRequest(Message message, ActivationData targetActivation) } internal void ProcessRequestToInvalidActivation( - Message message, - ActivationAddress oldAddress, - ActivationAddress forwardingAddress, - string failedOperation, + Message message, + ActivationAddress oldAddress, + ActivationAddress forwardingAddress, + string failedOperation, Exception exc = null) { - // Just use this opportunity to invalidate local Cache Entry as well. + // Just use this opportunity to invalidate local Cache Entry as well. if (oldAddress != null) { this.localGrainDirectory.InvalidateCacheEntry(oldAddress); @@ -484,12 +521,12 @@ internal void ProcessRequestToInvalidActivation( internal void ProcessRequestsToInvalidActivation( List messages, ActivationAddress oldAddress, - ActivationAddress forwardingAddress, + ActivationAddress forwardingAddress, string failedOperation, Exception exc = null, bool rejectMessages = false) { - // Just use this opportunity to invalidate local Cache Entry as well. + // Just use this opportunity to invalidate local Cache Entry as well. if (oldAddress != null) { this.localGrainDirectory.InvalidateCacheEntry(oldAddress); @@ -515,7 +552,7 @@ internal void ProcessRequestsToInvalidActivation( { TryForwardRequest(message, oldAddress, forwardingAddress, failedOperation, exc); } - + } } ), catalog.SchedulingContext); @@ -632,7 +669,7 @@ private static bool MayForward(Message message, SiloMessagingOptions messagingOp /// - may buffer for transaction completion / commit if it ends a transaction /// - choose target placement address, maintaining send order /// - add ordering info and maintain send order - /// + /// /// /// /// @@ -778,7 +815,7 @@ internal void SendResponse(Message request, Response response) internal void SendSystemTargetMessage(Message message) { - message.Category = message.TargetGrain.Equals(Constants.MembershipOracleId) ? + message.Category = message.TargetGrain.Equals(Constants.MembershipOracleId) ? Message.Categories.Ping : Message.Categories.System; if (message.TargetSilo == null) @@ -809,7 +846,7 @@ public void TransportMessage(Message message, ActivationData sendingActivation = /// Invoked when an activation has finished a transaction and may be ready for additional transactions /// /// The activation that has just completed processing this message - /// The message that has just completed processing. + /// The message that has just completed processing. /// This will be null for the case of completion of Activate/Deactivate calls. internal void OnActivationCompletedRequest(ActivationData activation, Message message) { @@ -856,7 +893,7 @@ internal void RunMessagePump(ActivationData activation) var nextMessage = activation.PeekNextWaitingMessage(); if (nextMessage == null) continue; if (!ActivationMayAcceptRequest(activation, nextMessage)) continue; - + activation.DequeueNextWaitingMessage(); // we might be over-writing an already running read only request. HandleIncomingRequest(nextMessage, activation); diff --git a/src/Orleans.Runtime/Core/InsideRuntimeClient.cs b/src/Orleans.Runtime/Core/InsideRuntimeClient.cs index 7f94c24ba8e..2f60d260f6a 100644 --- a/src/Orleans.Runtime/Core/InsideRuntimeClient.cs +++ b/src/Orleans.Runtime/Core/InsideRuntimeClient.cs @@ -99,7 +99,7 @@ public InsideRuntimeClient( private SiloAddress MySilo { get; } public GrainFactory ConcreteGrainFactory { get; } - + private Catalog Catalog => this.catalog ?? (this.catalog = this.ServiceProvider.GetRequiredService()); private ILocalGrainDirectory Directory @@ -252,12 +252,12 @@ public void SniffIncomingMessage(Message message) //// 1: //// Also record sending activation address for responses only in the cache. //// We don't record sending addresses for requests, since it is not clear that this silo ever wants to send messages to the grain sending this request. - //// However, it is sure that this silo does send messages to the sender of a reply. + //// However, it is sure that this silo does send messages to the sender of a reply. //// In most cases it will already have its address cached, unless it had a wrong outdated address cached and now this is a fresher address. //// It is anyway always safe to cache the replier address. - //// 2: + //// 2: //// after further thought decided not to do it. - //// It seems to better not bother caching the sender of a response at all, + //// It seems to better not bother caching the sender of a response at all, //// and instead to take a very occasional hit of a full remote look-up instead of this small but non-zero hit on every response. //if (message.Direction.Equals(Message.Directions.Response) && message.Result.Equals(Message.ResponseTypes.Success)) //{ @@ -289,9 +289,15 @@ public async Task Invoke(IAddressable target, IInvokable invokable, Message mess } RequestContextExtensions.Import(message.RequestContextData); - if (schedulingOptions.PerformDeadlockDetection && !message.TargetGrain.IsSystemTarget) + if ((schedulingOptions.PerformDeadlockDetection + || schedulingOptions.CallChainReentrancy == SchedulingOptions.CallChainReentrancyStrategy.EntireChain) + && !message.TargetGrain.IsSystemTarget) { - UpdateDeadlockInfoInRequestContext(new RequestInvocationHistory(message.TargetGrain, message.TargetActivation, message.DebugContext)); + RequestInvocationHistorySummary invocationHistory = !schedulingOptions.PerformDeadlockDetection ? + new RequestInvocationHistorySummary(message.TargetActivation) : + new RequestInvocationHistory(message.TargetGrain, message.TargetActivation, message.DebugContext); + + UpdateInvocationHistoryInRequestContext(invocationHistory); // RequestContext is automatically saved in the msg upon send and propagated to the next hop // in RuntimeClient.CreateMessage -> RequestContextExtensions.ExportToMessage(message); } @@ -329,7 +335,7 @@ public async Task Invoke(IAddressable target, IInvokable invokable, Message mess !(target is IGrainExtension) && !TryInstallExtension(request.InterfaceId, invokable, message.GenericGrainType, ref invoker)) { - // We are trying the invoke a grain extension method on a grain + // We are trying the invoke a grain extension method on a grain // -- most likely reason is that the dynamic extension is not installed for this grain // So throw a specific exception here rather than a general InvalidCastException var error = String.Format( @@ -345,7 +351,7 @@ public async Task Invoke(IAddressable target, IInvokable invokable, Message mess throw exc; } - + var requestInvoker = new GrainMethodInvoker(target, request, invoker, GrainCallFilters, interfaceToImplementationMapping); await requestInvoker.Invoke(); resultObject = requestInvoker.Result; @@ -366,7 +372,7 @@ public async Task Invoke(IAddressable target, IInvokable invokable, Message mess if (transactionInfo != null) { transactionInfo.ReconcilePending(); - + // Record reason for abort, if not alread set transactionInfo.RecordException(exc1, serializationManager); @@ -508,7 +514,7 @@ private void SafeSendResponse(Message message, object resultObject) private static readonly Lazy> prepForRemotingLazy = new Lazy>(CreateExceptionPrepForRemotingMethod); - + private static Func CreateExceptionPrepForRemotingMethod() { var methodInfo = typeof(Exception).GetMethod( @@ -568,19 +574,20 @@ private void SafeSendExceptionResponse(Message message, Exception ex) } // assumes deadlock information was already loaded into RequestContext from the message - private static void UpdateDeadlockInfoInRequestContext(RequestInvocationHistory thisInvocation) + private static void UpdateInvocationHistoryInRequestContext(RequestInvocationHistorySummary thisInvocation) { - IList prevChain; - object obj = RequestContext.Get(RequestContext.CALL_CHAIN_REQUEST_CONTEXT_HEADER); - if (obj != null) + IList prevChain; + + if (RequestContext.Get(RequestContext.CALL_CHAIN_REQUEST_CONTEXT_HEADER) is IList obj) { - prevChain = ((IList)obj); + prevChain = obj; } else { - prevChain = new List(); + prevChain = new List(); RequestContext.Set(RequestContext.CALL_CHAIN_REQUEST_CONTEXT_HEADER, prevChain); } + // append this call to the end of the call chain. Update in place. prevChain.Add(thisInvocation); } @@ -612,7 +619,7 @@ public void ReceiveResponse(Message message) if (message.CacheInvalidationHeader == null) { // Remove from local directory cache. Note that SendingGrain is the original target, since message is the rejection response. - // If CacheMgmtHeader is present, we already did this. Otherwise, we left this code for backward compatability. + // If CacheMgmtHeader is present, we already did this. Otherwise, we left this code for backward compatability. // It should be retired as we move to use CacheMgmtHeader in all relevant places. this.Directory.InvalidateCacheEntry(message.SendingAddress); } @@ -635,7 +642,7 @@ public void ReceiveResponse(Message message) callbackData.TransactionInfo.Join(message.TransactionInfo); } // IMPORTANT: we do not schedule the response callback via the scheduler, since the only thing it does - // is to resolve/break the resolver. The continuations/waits that are based on this resolution will be scheduled as work items. + // is to resolve/break the resolver. The continuations/waits that are based on this resolution will be scheduled as work items. callbackData.DoCallback(message); } else diff --git a/test/TesterInternal/AllowCallChainReentrancyEntireChainTests.cs b/test/TesterInternal/AllowCallChainReentrancyEntireChainTests.cs new file mode 100644 index 00000000000..1803602f998 --- /dev/null +++ b/test/TesterInternal/AllowCallChainReentrancyEntireChainTests.cs @@ -0,0 +1,169 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Orleans.Configuration; +using Orleans.Hosting; +using Orleans.TestingHost; +using TestExtensions; +using UnitTests.GrainInterfaces; +using Xunit; + +namespace UnitTests.General +{ + public class AllowCallChainReentrancyEntireChainTests : OrleansTestingBase, + IClassFixture + { + private readonly Fixture fixture; + + public class Fixture : BaseTestClusterFixture + { + protected override void ConfigureTestCluster(TestClusterBuilder builder) + { + builder.AddSiloBuilderConfigurator(); + } + + private class SiloConfigurator : ISiloBuilderConfigurator + { + public void Configure(ISiloHostBuilder hostBuilder) + { + hostBuilder.Configure(options => + { + options.PerformDeadlockDetection = false; + options.CallChainReentrancy = SchedulingOptions.CallChainReentrancyStrategy.EntireChain; + }); + } + } + + } + + private const int numIterations = 1; + + + public AllowCallChainReentrancyEntireChainTests(Fixture fixture) + { + this.fixture = fixture; + } + + // TODO combine tests since they are identical + + // 2 silos, loop across all cases (to force all grains to be local and remote): + // Non Reentrant A, B, C + // Reentrant X + // 1) No Deadlock A, A + // 2) No Deadlock A, B, A + // 3) No Deadlock X, A, X, A + // 4) No Deadlock X, X + // 5) No Deadlock X, A, X + // 6) No Deadlock A, B, C, A + + // 1) Allowed reentrancy A, A + [Fact, TestCategory("Functional"), TestCategory("Deadlock")] + public async Task DeadlockDetection_1() + { + long baseGrainId = random.Next(); + for (int i = 0; i < numIterations; i++) + { + long grainId = baseGrainId + i; + IDeadlockNonReentrantGrain firstGrain = this.fixture.GrainFactory.GetGrain(grainId); + List> callChain = new List>(); + callChain.Add(new Tuple(grainId, true)); + callChain.Add(new Tuple(grainId, true)); + await firstGrain.CallNext_1(callChain, 1); + } + } + + // 2) Allowed reentrancy on non-reentrant grains A, B, A + [Fact, TestCategory("Functional"), TestCategory("Deadlock")] + public async Task DeadlockDetection_2() + { + long baseGrainId = random.Next(); + long bBase = 100; + for (int i = 0; i < numIterations; i++) + { + long grainId = baseGrainId + i; + IDeadlockNonReentrantGrain firstGrain = this.fixture.GrainFactory.GetGrain(grainId); + List> callChain = new List>(); + callChain.Add(new Tuple(grainId, true)); + callChain.Add(new Tuple(bBase + grainId, true)); + callChain.Add(new Tuple(grainId, true)); + await firstGrain.CallNext_1(callChain, 1); + } + } + + // 3) Allowed reentrancy X, A, X, A + [Fact, TestCategory("Functional"), TestCategory("Deadlock")] + public async Task DeadlockDetection_3() + { + long baseGrainId = random.Next(); + long xBase = 1000; + for (int i = 0; i < numIterations; i++) + { + long grainId = baseGrainId + i; + IDeadlockReentrantGrain firstGrain = this.fixture.GrainFactory.GetGrain(grainId); + List> callChain = new List>(); + callChain.Add(new Tuple(xBase + grainId, false)); + callChain.Add(new Tuple(grainId, true)); + callChain.Add(new Tuple(xBase + grainId, false)); + callChain.Add(new Tuple(grainId, true)); + await firstGrain.CallNext_1(callChain, 1); + } + } + + // 4) No Deadlock X, X + [Fact, TestCategory("Functional"), TestCategory("Deadlock")] + public async Task DeadlockDetection_4() + { + long baseGrainId = random.Next(); + long xBase = 1000; + for (int i = 0; i < numIterations; i++) + { + long grainId = baseGrainId + i; + IDeadlockReentrantGrain firstGrain = this.fixture.GrainFactory.GetGrain(grainId); + List> callChain = new List>(); + callChain.Add(new Tuple(xBase + grainId, false)); + callChain.Add(new Tuple(xBase + grainId, false)); + + await firstGrain.CallNext_1(callChain, 1); + } + } + + // 5) No Deadlock X, A, X + [Fact, TestCategory("Functional"), TestCategory("Deadlock")] + public async Task DeadlockDetection_5() + { + long baseGrainId = random.Next(); + long xBase = 1000; + for (int i = 0; i < numIterations; i++) + { + long grainId = baseGrainId + i; + IDeadlockReentrantGrain firstGrain = this.fixture.GrainFactory.GetGrain(grainId); + List> callChain = new List>(); + callChain.Add(new Tuple(xBase + grainId, false)); + callChain.Add(new Tuple(grainId, true)); + callChain.Add(new Tuple(xBase + grainId, false)); + + await firstGrain.CallNext_1(callChain, 1); + } + } + + // 6) Allowed reentrancy on non-reentrant grains A, B, C, A + [Fact, TestCategory("Functional"), TestCategory("Deadlock")] + public async Task DeadlockDetection_6() + { + long baseGrainId = random.Next(); + long bBase = 100; + long cBase = 200; + for (int i = 0; i < numIterations; i++) + { + long grainId = baseGrainId + i; + IDeadlockNonReentrantGrain firstGrain = this.fixture.GrainFactory.GetGrain(grainId); + List> callChain = new List>(); + callChain.Add(new Tuple(grainId, true)); + callChain.Add(new Tuple(bBase + grainId, true)); + callChain.Add(new Tuple(cBase + grainId, true)); + callChain.Add(new Tuple(grainId, true)); + await firstGrain.CallNext_1(callChain, 1); + } + } + } +} \ No newline at end of file diff --git a/test/TesterInternal/DeadlockDetectionWithAllowCallChainReentrancyTests.cs b/test/TesterInternal/DeadlockDetectionWithAllowCallChainSingleCallReentrancyTests.cs similarity index 82% rename from test/TesterInternal/DeadlockDetectionWithAllowCallChainReentrancyTests.cs rename to test/TesterInternal/DeadlockDetectionWithAllowCallChainSingleCallReentrancyTests.cs index 95f6732e12f..3d05651a460 100644 --- a/test/TesterInternal/DeadlockDetectionWithAllowCallChainReentrancyTests.cs +++ b/test/TesterInternal/DeadlockDetectionWithAllowCallChainSingleCallReentrancyTests.cs @@ -11,7 +11,7 @@ namespace UnitTests.General { - public class DeadlockDetectionWithAllowCallChainReentrancyTests : OrleansTestingBase, IClassFixture + public class DeadlockDetectionWithAllowCallChainSingleCallReentrancyTests : OrleansTestingBase, IClassFixture { private readonly Fixture fixture; @@ -29,7 +29,7 @@ public void Configure(ISiloHostBuilder hostBuilder) hostBuilder.Configure(options => { options.PerformDeadlockDetection = true; - options.AllowCallChainReentrancy = true; + options.CallChainReentrancy = SchedulingOptions.CallChainReentrancyStrategy.SingleCall; }); } } @@ -38,20 +38,20 @@ public void Configure(ISiloHostBuilder hostBuilder) private const int numIterations = 30; - public DeadlockDetectionWithAllowCallChainReentrancyTests(Fixture fixture) + public DeadlockDetectionWithAllowCallChainSingleCallReentrancyTests(Fixture fixture) { this.fixture = fixture; } // 2 silos, loop across all cases (to force all grains to be local and remote): - // Non Reentrant A, B, D - // Reentrant C + // Non Reentrant A, B, C + // Reentrant X // 1) No Deadlock A, A // 2) No Deadlock A, B, A - // 3) No Deadlock C, A, C, A - // 4) No Deadlock C, C - // 5) No Deadlock C, A, C - // 6) No Deadlock A, B, D, A + // 3) No Deadlock X, A, X, A + // 4) No Deadlock X, X + // 5) No Deadlock X, A, X + // 6) No Deadlock A, B, C, A // 1) Allowed reentrancy A, A [Fact, TestCategory("Functional"), TestCategory("Deadlock")] @@ -87,69 +87,69 @@ public async Task DeadlockDetection_2() } } - // 3) Allowed reentrancy C, A, C, A + // 3) Allowed reentrancy X, A, X, A [Fact, TestCategory("Functional"), TestCategory("Deadlock")] public async Task DeadlockDetection_3() { long baseGrainId = random.Next(); - long cBase = 200; + long xBase = 1000; for (int i = 0; i < numIterations; i++) { long grainId = baseGrainId + i; IDeadlockReentrantGrain firstGrain = this.fixture.GrainFactory.GetGrain(grainId); List> callChain = new List>(); - callChain.Add(new Tuple(cBase + grainId, false)); + callChain.Add(new Tuple(xBase + grainId, false)); callChain.Add(new Tuple(grainId, true)); - callChain.Add(new Tuple(cBase + grainId, false)); + callChain.Add(new Tuple(xBase + grainId, false)); callChain.Add(new Tuple(grainId, true)); await firstGrain.CallNext_1(callChain, 1); } } - // 4) No Deadlock C, C + // 4) No Deadlock X, X [Fact, TestCategory("Functional"), TestCategory("Deadlock")] public async Task DeadlockDetection_4() { long baseGrainId = random.Next(); - long cBase = 200; + long xBase = 1000; for (int i = 0; i < numIterations; i++) { long grainId = baseGrainId + i; IDeadlockReentrantGrain firstGrain = this.fixture.GrainFactory.GetGrain(grainId); List> callChain = new List>(); - callChain.Add(new Tuple(cBase + grainId, false)); - callChain.Add(new Tuple(cBase + grainId, false)); + callChain.Add(new Tuple(xBase + grainId, false)); + callChain.Add(new Tuple(xBase + grainId, false)); await firstGrain.CallNext_1(callChain, 1); } } - // 5) No Deadlock C, A, C + // 5) No Deadlock X, A, X [Fact, TestCategory("Functional"), TestCategory("Deadlock")] public async Task DeadlockDetection_5() { long baseGrainId = random.Next(); - long cBase = 200; + long xBase = 1000; for (int i = 0; i < numIterations; i++) { long grainId = baseGrainId + i; IDeadlockReentrantGrain firstGrain = this.fixture.GrainFactory.GetGrain(grainId); List> callChain = new List>(); - callChain.Add(new Tuple(cBase + grainId, false)); + callChain.Add(new Tuple(xBase + grainId, false)); callChain.Add(new Tuple(grainId, true)); - callChain.Add(new Tuple(cBase + grainId, false)); + callChain.Add(new Tuple(xBase + grainId, false)); await firstGrain.CallNext_1(callChain, 1); } } - // 6) Allowed reentrancy on non-reentrant grains A, B, D, A + // 6) Allowed reentrancy on non-reentrant grains A, B, C, A [Fact, TestCategory("Functional"), TestCategory("Deadlock")] public async Task DeadlockDetection_6() { long baseGrainId = random.Next(); long bBase = 100; - long dBase = 200; + long cBase = 200; for (int i = 0; i < numIterations; i++) { long grainId = baseGrainId + i; @@ -157,7 +157,7 @@ public async Task DeadlockDetection_6() List> callChain = new List>(); callChain.Add(new Tuple(grainId, true)); callChain.Add(new Tuple(bBase + grainId, true)); - callChain.Add(new Tuple(dBase + grainId, true)); + callChain.Add(new Tuple(cBase + grainId, true)); callChain.Add(new Tuple(grainId, true)); await firstGrain.CallNext_1(callChain, 1); } diff --git a/test/TesterInternal/DeadlockDetectionWithoutAllowCallChainReentrancyTests.cs b/test/TesterInternal/DeadlockDetectionWithoutAllowCallChainReentrancyTests.cs index ca4a9744ee5..77fa4284b0c 100644 --- a/test/TesterInternal/DeadlockDetectionWithoutAllowCallChainReentrancyTests.cs +++ b/test/TesterInternal/DeadlockDetectionWithoutAllowCallChainReentrancyTests.cs @@ -30,7 +30,7 @@ public void Configure(ISiloHostBuilder hostBuilder) hostBuilder.Configure(options => { options.PerformDeadlockDetection = true; - options.AllowCallChainReentrancy = false; + options.CallChainReentrancy = SchedulingOptions.CallChainReentrancyStrategy.None; }); } } @@ -44,6 +44,8 @@ public DeadlockDetectionWithoutAllowCallChainReentrancyTests(Fixture fixture) this.fixture = fixture; } + // TODO add test 6 + // 2 silos, loop across all cases (to force all grains to be local and remote): // Non Reentrant A, B // Reentrant C From 429bc323ec68a74e8d25f10aa9f4216622eaf705 Mon Sep 17 00:00:00 2001 From: Andrew Hesky Date: Mon, 22 Oct 2018 21:25:40 -0700 Subject: [PATCH 03/10] Logging works, but sucks. Why doesn't the original call handle the response? --- src/Orleans.Runtime/Core/Dispatcher.cs | 3 +- test/Grains/TestGrains/DeadlockGrain.cs | 14 +- ...llowCallChainReentrancyEntireChainTests.cs | 123 ++++------------- .../CallChainReentrancyTestHelper.cs | 129 ++++++++++++++++++ ...AllowCallChainSingleCallReentrancyTests.cs | 102 ++------------ test/TesterInternal/TestLoggingProvider.cs | 46 +++++++ 6 files changed, 226 insertions(+), 191 deletions(-) create mode 100644 test/TesterInternal/CallChainReentrancyTestHelper.cs create mode 100644 test/TesterInternal/TestLoggingProvider.cs diff --git a/src/Orleans.Runtime/Core/Dispatcher.cs b/src/Orleans.Runtime/Core/Dispatcher.cs index 259e136aab5..d12ce865678 100644 --- a/src/Orleans.Runtime/Core/Dispatcher.cs +++ b/src/Orleans.Runtime/Core/Dispatcher.cs @@ -400,8 +400,7 @@ private bool IsMessageACallChainLoop(Message message) /// Message to analyze private void CheckDeadlock(Message message) { - if (IsMessageACallChainLoop(message) - && schedulingOptions.CallChainReentrancy != SchedulingOptions.CallChainReentrancyStrategy.EntireChain) + if (IsMessageACallChainLoop(message)) { IEnumerable> callChain = Enumerable.Empty>(); string exceptionMessage = string.Empty; diff --git a/test/Grains/TestGrains/DeadlockGrain.cs b/test/Grains/TestGrains/DeadlockGrain.cs index 432b954e815..4b5fae35646 100644 --- a/test/Grains/TestGrains/DeadlockGrain.cs +++ b/test/Grains/TestGrains/DeadlockGrain.cs @@ -38,16 +38,18 @@ public class DeadlockNonReentrantGrain : Grain, IDeadlockNonReentrantGrain { private string Id { get { return String.Format("DeadlockNonReentrantGrain {0}", base.IdentityString); } } - public Task CallNext_1(List> callChain, int currCallIndex) + public async Task CallNext_1(List> callChain, int currCallIndex) { - this.GetLogger(Id).Info("Inside grain {0} CallNext_1().", Id); - return DeadlockGrain.CallNext(GrainFactory, callChain, currCallIndex); + this.GetLogger(Id).Info("Inside grain {0}.{1} CallNext_1().", Id, currCallIndex); + await DeadlockGrain.CallNext(GrainFactory, callChain, currCallIndex); + this.GetLogger(Id).Info("Inside grain {0}.{1} CallNext_1() Finished.", Id, currCallIndex); } - public Task CallNext_2(List> callChain, int currCallIndex) + public async Task CallNext_2(List> callChain, int currCallIndex) { - this.GetLogger(Id).Info("Inside grain {0} CallNext_2().", Id); - return DeadlockGrain.CallNext(GrainFactory, callChain, currCallIndex); + this.GetLogger(Id).Info("Inside grain {0}.{1} CallNext_2().", Id, currCallIndex); + await DeadlockGrain.CallNext(GrainFactory, callChain, currCallIndex); + this.GetLogger(Id).Info("Inside grain {0}.{1} CallNext_2() Finished.", Id, currCallIndex); } } diff --git a/test/TesterInternal/AllowCallChainReentrancyEntireChainTests.cs b/test/TesterInternal/AllowCallChainReentrancyEntireChainTests.cs index 1803602f998..4260e7b988b 100644 --- a/test/TesterInternal/AllowCallChainReentrancyEntireChainTests.cs +++ b/test/TesterInternal/AllowCallChainReentrancyEntireChainTests.cs @@ -1,22 +1,24 @@ using System; -using System.Collections.Generic; using System.Threading.Tasks; +using Microsoft.Extensions.Logging; using Orleans.Configuration; using Orleans.Hosting; using Orleans.TestingHost; using TestExtensions; -using UnitTests.GrainInterfaces; using Xunit; +using Xunit.Abstractions; namespace UnitTests.General { - public class AllowCallChainReentrancyEntireChainTests : OrleansTestingBase, - IClassFixture + public class AllowCallChainReentrancyEntireChainTests : OrleansTestingBase { private readonly Fixture fixture; public class Fixture : BaseTestClusterFixture { + // TODO static is really not what we want here + public static ITestOutputHelper _output { get; set; } + protected override void ConfigureTestCluster(TestClusterBuilder builder) { builder.AddSiloBuilderConfigurator(); @@ -28,142 +30,75 @@ public void Configure(ISiloHostBuilder hostBuilder) { hostBuilder.Configure(options => { - options.PerformDeadlockDetection = false; + options.PerformDeadlockDetection = true; options.CallChainReentrancy = SchedulingOptions.CallChainReentrancyStrategy.EntireChain; + }).ConfigureLogging(logging => + { + logging.AddProvider(new TestLoggingProvider(x => _output.WriteLine(x))); }); + } } - } private const int numIterations = 1; + private readonly CallChainReentrancyTestHelper testHelper; - public AllowCallChainReentrancyEntireChainTests(Fixture fixture) + public AllowCallChainReentrancyEntireChainTests(ITestOutputHelper output) { - this.fixture = fixture; - } - - // TODO combine tests since they are identical + if(output == null) throw new ArgumentNullException(nameof(output)); - // 2 silos, loop across all cases (to force all grains to be local and remote): - // Non Reentrant A, B, C - // Reentrant X - // 1) No Deadlock A, A - // 2) No Deadlock A, B, A - // 3) No Deadlock X, A, X, A - // 4) No Deadlock X, X - // 5) No Deadlock X, A, X - // 6) No Deadlock A, B, C, A + Fixture._output = output; + this.fixture = new Fixture(); + testHelper = new CallChainReentrancyTestHelper + { + Random = random, + Fixture = fixture, + NumIterations = numIterations + }; + } // 1) Allowed reentrancy A, A [Fact, TestCategory("Functional"), TestCategory("Deadlock")] public async Task DeadlockDetection_1() { - long baseGrainId = random.Next(); - for (int i = 0; i < numIterations; i++) - { - long grainId = baseGrainId + i; - IDeadlockNonReentrantGrain firstGrain = this.fixture.GrainFactory.GetGrain(grainId); - List> callChain = new List>(); - callChain.Add(new Tuple(grainId, true)); - callChain.Add(new Tuple(grainId, true)); - await firstGrain.CallNext_1(callChain, 1); - } + await testHelper.DeadlockDetection_1(); } // 2) Allowed reentrancy on non-reentrant grains A, B, A [Fact, TestCategory("Functional"), TestCategory("Deadlock")] public async Task DeadlockDetection_2() { - long baseGrainId = random.Next(); - long bBase = 100; - for (int i = 0; i < numIterations; i++) - { - long grainId = baseGrainId + i; - IDeadlockNonReentrantGrain firstGrain = this.fixture.GrainFactory.GetGrain(grainId); - List> callChain = new List>(); - callChain.Add(new Tuple(grainId, true)); - callChain.Add(new Tuple(bBase + grainId, true)); - callChain.Add(new Tuple(grainId, true)); - await firstGrain.CallNext_1(callChain, 1); - } + await testHelper.DeadlockDetection_2(); } // 3) Allowed reentrancy X, A, X, A [Fact, TestCategory("Functional"), TestCategory("Deadlock")] public async Task DeadlockDetection_3() { - long baseGrainId = random.Next(); - long xBase = 1000; - for (int i = 0; i < numIterations; i++) - { - long grainId = baseGrainId + i; - IDeadlockReentrantGrain firstGrain = this.fixture.GrainFactory.GetGrain(grainId); - List> callChain = new List>(); - callChain.Add(new Tuple(xBase + grainId, false)); - callChain.Add(new Tuple(grainId, true)); - callChain.Add(new Tuple(xBase + grainId, false)); - callChain.Add(new Tuple(grainId, true)); - await firstGrain.CallNext_1(callChain, 1); - } + await testHelper.DeadlockDetection_3(); } // 4) No Deadlock X, X [Fact, TestCategory("Functional"), TestCategory("Deadlock")] public async Task DeadlockDetection_4() { - long baseGrainId = random.Next(); - long xBase = 1000; - for (int i = 0; i < numIterations; i++) - { - long grainId = baseGrainId + i; - IDeadlockReentrantGrain firstGrain = this.fixture.GrainFactory.GetGrain(grainId); - List> callChain = new List>(); - callChain.Add(new Tuple(xBase + grainId, false)); - callChain.Add(new Tuple(xBase + grainId, false)); - - await firstGrain.CallNext_1(callChain, 1); - } + await testHelper.DeadlockDetection_4(); } // 5) No Deadlock X, A, X [Fact, TestCategory("Functional"), TestCategory("Deadlock")] public async Task DeadlockDetection_5() { - long baseGrainId = random.Next(); - long xBase = 1000; - for (int i = 0; i < numIterations; i++) - { - long grainId = baseGrainId + i; - IDeadlockReentrantGrain firstGrain = this.fixture.GrainFactory.GetGrain(grainId); - List> callChain = new List>(); - callChain.Add(new Tuple(xBase + grainId, false)); - callChain.Add(new Tuple(grainId, true)); - callChain.Add(new Tuple(xBase + grainId, false)); - - await firstGrain.CallNext_1(callChain, 1); - } + await testHelper.DeadlockDetection_5(); } // 6) Allowed reentrancy on non-reentrant grains A, B, C, A [Fact, TestCategory("Functional"), TestCategory("Deadlock")] public async Task DeadlockDetection_6() { - long baseGrainId = random.Next(); - long bBase = 100; - long cBase = 200; - for (int i = 0; i < numIterations; i++) - { - long grainId = baseGrainId + i; - IDeadlockNonReentrantGrain firstGrain = this.fixture.GrainFactory.GetGrain(grainId); - List> callChain = new List>(); - callChain.Add(new Tuple(grainId, true)); - callChain.Add(new Tuple(bBase + grainId, true)); - callChain.Add(new Tuple(cBase + grainId, true)); - callChain.Add(new Tuple(grainId, true)); - await firstGrain.CallNext_1(callChain, 1); - } + await testHelper.DeadlockDetection_6(); } } } \ No newline at end of file diff --git a/test/TesterInternal/CallChainReentrancyTestHelper.cs b/test/TesterInternal/CallChainReentrancyTestHelper.cs new file mode 100644 index 00000000000..46a10905331 --- /dev/null +++ b/test/TesterInternal/CallChainReentrancyTestHelper.cs @@ -0,0 +1,129 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using TestExtensions; +using UnitTests.GrainInterfaces; + +namespace UnitTests.General +{ + public class CallChainReentrancyTestHelper + { + public Random Random { get; set; } + public BaseTestClusterFixture Fixture { get; set; } + public int NumIterations { get; set; } + + // 2 silos, loop across all cases (to force all grains to be local and remote): + // Non Reentrant A, B, C + // Reentrant X + // 1) No Deadlock A, A + // 2) No Deadlock A, B, A + // 3) No Deadlock X, A, X, A + // 4) No Deadlock X, X + // 5) No Deadlock X, A, X + // 6) No Deadlock A, B, C, A + + // 1) Allowed reentrancy A, A + public async Task DeadlockDetection_1() + { + long baseGrainId = this.Random.Next(); + for (int i = 0; i < this.NumIterations; i++) + { + long grainId = baseGrainId + i; + IDeadlockNonReentrantGrain firstGrain = this.Fixture.GrainFactory.GetGrain(grainId); + List> callChain = new List>(); + callChain.Add(new Tuple(grainId, true)); + callChain.Add(new Tuple(grainId, true)); + await firstGrain.CallNext_1(callChain, 1); + } + } + + // 2) Allowed reentrancy on non-reentrant grains A, B, A + public async Task DeadlockDetection_2() + { + long baseGrainId = this.Random.Next(); + long bBase = 100; + for (int i = 0; i < this.NumIterations; i++) + { + long grainId = baseGrainId + i; + IDeadlockNonReentrantGrain firstGrain = this.Fixture.GrainFactory.GetGrain(grainId); + List> callChain = new List>(); + callChain.Add(new Tuple(grainId, true)); + callChain.Add(new Tuple(bBase + grainId, true)); + callChain.Add(new Tuple(grainId, true)); + await firstGrain.CallNext_1(callChain, 1); + } + } + + // 3) Allowed reentrancy X, A, X, A + public async Task DeadlockDetection_3() + { + long baseGrainId = this.Random.Next(); + long xBase = 1000; + for (int i = 0; i < this.NumIterations; i++) + { + long grainId = baseGrainId + i; + IDeadlockReentrantGrain firstGrain = this.Fixture.GrainFactory.GetGrain(grainId); + List> callChain = new List>(); + callChain.Add(new Tuple(xBase + grainId, false)); + callChain.Add(new Tuple(grainId, true)); + callChain.Add(new Tuple(xBase + grainId, false)); + callChain.Add(new Tuple(grainId, true)); + await firstGrain.CallNext_1(callChain, 1); + } + } + + // 4) No Deadlock X, X + public async Task DeadlockDetection_4() + { + long baseGrainId = this.Random.Next(); + long xBase = 1000; + for (int i = 0; i < this.NumIterations; i++) + { + long grainId = baseGrainId + i; + IDeadlockReentrantGrain firstGrain = this.Fixture.GrainFactory.GetGrain(grainId); + List> callChain = new List>(); + callChain.Add(new Tuple(xBase + grainId, false)); + callChain.Add(new Tuple(xBase + grainId, false)); + + await firstGrain.CallNext_1(callChain, 1); + } + } + + // 5) No Deadlock X, A, X + public async Task DeadlockDetection_5() + { + long baseGrainId = this.Random.Next(); + long xBase = 1000; + for (int i = 0; i < this.NumIterations; i++) + { + long grainId = baseGrainId + i; + IDeadlockReentrantGrain firstGrain = this.Fixture.GrainFactory.GetGrain(grainId); + List> callChain = new List>(); + callChain.Add(new Tuple(xBase + grainId, false)); + callChain.Add(new Tuple(grainId, true)); + callChain.Add(new Tuple(xBase + grainId, false)); + + await firstGrain.CallNext_1(callChain, 1); + } + } + + // 6) Allowed reentrancy on non-reentrant grains only when using full chain reentrancy A, B, C, A + public async Task DeadlockDetection_6() + { + long baseGrainId = this.Random.Next(); + long bBase = 100; + long cBase = 200; + for (int i = 0; i < this.NumIterations; i++) + { + long grainId = baseGrainId + i; + IDeadlockNonReentrantGrain firstGrain = this.Fixture.GrainFactory.GetGrain(grainId); + List> callChain = new List>(); + callChain.Add(new Tuple(grainId, true)); + callChain.Add(new Tuple(bBase + grainId, true)); + callChain.Add(new Tuple(cBase + grainId, true)); + callChain.Add(new Tuple(grainId, true)); + await firstGrain.CallNext_1(callChain, 1); + } + } + } +} \ No newline at end of file diff --git a/test/TesterInternal/DeadlockDetectionWithAllowCallChainSingleCallReentrancyTests.cs b/test/TesterInternal/DeadlockDetectionWithAllowCallChainSingleCallReentrancyTests.cs index 3d05651a460..cc19b96d7f9 100644 --- a/test/TesterInternal/DeadlockDetectionWithAllowCallChainSingleCallReentrancyTests.cs +++ b/test/TesterInternal/DeadlockDetectionWithAllowCallChainSingleCallReentrancyTests.cs @@ -38,129 +38,53 @@ public void Configure(ISiloHostBuilder hostBuilder) private const int numIterations = 30; + private readonly CallChainReentrancyTestHelper testHelper; + public DeadlockDetectionWithAllowCallChainSingleCallReentrancyTests(Fixture fixture) { this.fixture = fixture; + testHelper = new CallChainReentrancyTestHelper() + { + Random = random, + Fixture = fixture, + NumIterations = numIterations + }; } - // 2 silos, loop across all cases (to force all grains to be local and remote): - // Non Reentrant A, B, C - // Reentrant X - // 1) No Deadlock A, A - // 2) No Deadlock A, B, A - // 3) No Deadlock X, A, X, A - // 4) No Deadlock X, X - // 5) No Deadlock X, A, X - // 6) No Deadlock A, B, C, A // 1) Allowed reentrancy A, A [Fact, TestCategory("Functional"), TestCategory("Deadlock")] public async Task DeadlockDetection_1() { - long baseGrainId = random.Next(); - for (int i = 0; i < numIterations; i++) - { - long grainId = baseGrainId + i; - IDeadlockNonReentrantGrain firstGrain = this.fixture.GrainFactory.GetGrain(grainId); - List> callChain = new List>(); - callChain.Add(new Tuple(grainId, true)); - callChain.Add(new Tuple(grainId, true)); - await firstGrain.CallNext_1(callChain, 1); - } + await testHelper.DeadlockDetection_1(); } // 2) Allowed reentrancy on non-reentrant grains A, B, A [Fact, TestCategory("Functional"), TestCategory("Deadlock")] public async Task DeadlockDetection_2() { - long baseGrainId = random.Next(); - long bBase = 100; - for (int i = 0; i < numIterations; i++) - { - long grainId = baseGrainId + i; - IDeadlockNonReentrantGrain firstGrain = this.fixture.GrainFactory.GetGrain(grainId); - List> callChain = new List>(); - callChain.Add(new Tuple(grainId, true)); - callChain.Add(new Tuple(bBase + grainId, true)); - callChain.Add(new Tuple(grainId, true)); - await firstGrain.CallNext_1(callChain, 1); - } + await testHelper.DeadlockDetection_2(); } // 3) Allowed reentrancy X, A, X, A [Fact, TestCategory("Functional"), TestCategory("Deadlock")] public async Task DeadlockDetection_3() { - long baseGrainId = random.Next(); - long xBase = 1000; - for (int i = 0; i < numIterations; i++) - { - long grainId = baseGrainId + i; - IDeadlockReentrantGrain firstGrain = this.fixture.GrainFactory.GetGrain(grainId); - List> callChain = new List>(); - callChain.Add(new Tuple(xBase + grainId, false)); - callChain.Add(new Tuple(grainId, true)); - callChain.Add(new Tuple(xBase + grainId, false)); - callChain.Add(new Tuple(grainId, true)); - await firstGrain.CallNext_1(callChain, 1); - } + await testHelper.DeadlockDetection_3(); } // 4) No Deadlock X, X [Fact, TestCategory("Functional"), TestCategory("Deadlock")] public async Task DeadlockDetection_4() { - long baseGrainId = random.Next(); - long xBase = 1000; - for (int i = 0; i < numIterations; i++) - { - long grainId = baseGrainId + i; - IDeadlockReentrantGrain firstGrain = this.fixture.GrainFactory.GetGrain(grainId); - List> callChain = new List>(); - callChain.Add(new Tuple(xBase + grainId, false)); - callChain.Add(new Tuple(xBase + grainId, false)); - - await firstGrain.CallNext_1(callChain, 1); - } + await testHelper.DeadlockDetection_4(); } // 5) No Deadlock X, A, X [Fact, TestCategory("Functional"), TestCategory("Deadlock")] public async Task DeadlockDetection_5() { - long baseGrainId = random.Next(); - long xBase = 1000; - for (int i = 0; i < numIterations; i++) - { - long grainId = baseGrainId + i; - IDeadlockReentrantGrain firstGrain = this.fixture.GrainFactory.GetGrain(grainId); - List> callChain = new List>(); - callChain.Add(new Tuple(xBase + grainId, false)); - callChain.Add(new Tuple(grainId, true)); - callChain.Add(new Tuple(xBase + grainId, false)); - - await firstGrain.CallNext_1(callChain, 1); - } - } - - // 6) Allowed reentrancy on non-reentrant grains A, B, C, A - [Fact, TestCategory("Functional"), TestCategory("Deadlock")] - public async Task DeadlockDetection_6() - { - long baseGrainId = random.Next(); - long bBase = 100; - long cBase = 200; - for (int i = 0; i < numIterations; i++) - { - long grainId = baseGrainId + i; - IDeadlockNonReentrantGrain firstGrain = this.fixture.GrainFactory.GetGrain(grainId); - List> callChain = new List>(); - callChain.Add(new Tuple(grainId, true)); - callChain.Add(new Tuple(bBase + grainId, true)); - callChain.Add(new Tuple(cBase + grainId, true)); - callChain.Add(new Tuple(grainId, true)); - await firstGrain.CallNext_1(callChain, 1); - } + await testHelper.DeadlockDetection_5(); } } } diff --git a/test/TesterInternal/TestLoggingProvider.cs b/test/TesterInternal/TestLoggingProvider.cs new file mode 100644 index 00000000000..1410410da0b --- /dev/null +++ b/test/TesterInternal/TestLoggingProvider.cs @@ -0,0 +1,46 @@ +using System; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions.Internal; + +namespace UnitTests.General +{ + public class TestLoggingProvider : ILoggerProvider + { + private readonly TestLogger logger; + + public TestLoggingProvider(Action logMethod) + { + this.logger = new TestLogger(logMethod); + } + + private class TestLogger : ILogger + { + private readonly Action logMethod; + + public TestLogger(Action logMethod) + { + this.logMethod = logMethod; + } + + void ILogger.Log(LogLevel logLevel, EventId eventId, TState state, Exception exception, + Func formatter) + { + this.logMethod($"{logLevel}: ({eventId}-{eventId})-{formatter(state, exception)}"); + } + + bool ILogger.IsEnabled(LogLevel logLevel) => true; + + IDisposable ILogger.BeginScope(TState state) => NullScope.Instance; + + } + + public void Dispose() + { + } + + public ILogger CreateLogger(string categoryName) + { + return this.logger; + } + } +} \ No newline at end of file From 70dddbd9d7cf40c425f9f63818200e3357ec83af Mon Sep 17 00:00:00 2001 From: Andrew Hesky Date: Mon, 29 Oct 2018 11:02:57 -0700 Subject: [PATCH 04/10] Handle deadlock detection at a higher level since response messages are not 'InvokeMethodRequest's which caused a cast exception --- src/Orleans.Runtime/Catalog/Catalog.cs | 52 +++++++++++++------------- src/Orleans.Runtime/Core/Dispatcher.cs | 10 +++-- 2 files changed, 32 insertions(+), 30 deletions(-) diff --git a/src/Orleans.Runtime/Catalog/Catalog.cs b/src/Orleans.Runtime/Catalog/Catalog.cs index 824b4d9ad34..6af84b183af 100644 --- a/src/Orleans.Runtime/Catalog/Catalog.cs +++ b/src/Orleans.Runtime/Catalog/Catalog.cs @@ -35,7 +35,7 @@ internal class NonExistentActivationException : Exception public NonExistentActivationException() : base("NonExistentActivationException") { } public NonExistentActivationException(string msg) : base(msg) { } - public NonExistentActivationException(string message, Exception innerException) + public NonExistentActivationException(string message, Exception innerException) : base(message, innerException) { } public NonExistentActivationException(string msg, ActivationAddress nonExistentActivation, bool isStatelessWorker) @@ -274,7 +274,7 @@ public List> GetGrainStatistics() // TODO: generic type expansion var grainTypeName = TypeUtils.GetFullName(data.GrainInstanceType); - + Dictionary grains; int n; if (!counts.TryGetValue(grainTypeName, out grains)) @@ -347,8 +347,8 @@ public DetailedGrainReport GetDetailedGrainReport(GrainId grain) } List acts = activations.FindTargets(grain); - report.LocalActivations = acts != null ? - acts.Select(activationData => activationData.ToDetailedString()).ToList() : + report.LocalActivations = acts != null ? + acts.Select(activationData => activationData.ToDetailedString()).ToList() : new List(); return report; } @@ -397,18 +397,18 @@ internal int UnregisterGrainForTesting(GrainId grain) int numActsBefore = acts.Count; foreach (var act in acts) UnregisterMessageTarget(act); - + return numActsBefore; } internal bool CanInterleave(ActivationId running, Message message) { - ActivationData target; - GrainTypeData data; - return TryGetActivationData(running, out target) && - target.GrainInstance != null && - GrainTypeManager.TryGetData(TypeUtils.GetFullName(target.GrainInstanceType), out data) && - (data.IsReentrant || data.MayInterleave((InvokeMethodRequest)message.GetDeserializedBody(this.serializationManager))); + return this.TryGetActivationData(running, out ActivationData target) + && target.GrainInstance != null + && this.GrainTypeManager.TryGetData(TypeUtils.GetFullName(target.GrainInstanceType), out GrainTypeData data) + && (data.IsReentrant + || (message.GetDeserializedBody(this.serializationManager) is InvokeMethodRequest invokeMethodRequest + && data.MayInterleave((invokeMethodRequest)))); } public void GetGrainTypeInfo(int typeCode, out string grainClass, out PlacementStrategy placement, out MultiClusterRegistrationStrategy activationStrategy, string genericArguments = null) @@ -449,7 +449,7 @@ public ActivationData GetOrCreateActivation( { return result; } - + int typeCode = address.Grain.TypeCode; string actualGrainType = null; MultiClusterRegistrationStrategy activationStrategy; @@ -502,7 +502,7 @@ public ActivationData GetOrCreateActivation( CounterStatistic.FindOrCreate(StatisticNames.CATALOG_ACTIVATION_NON_EXISTENT_ACTIVATIONS).Increment(); throw new NonExistentActivationException(msg, address, placement is StatelessWorkerPlacement); } - + SetupActivationInstance(result, grainType, genericArguments); activatedPromise = InitActivation(result, grainType, genericArguments, requestContextData); return result; @@ -714,7 +714,7 @@ private void CreateGrainInstance(string grainTypeName, ActivationData data, stri data.SetupContext(grainTypeData, this.serviceProvider); Grain grain = grainCreator.CreateGrainInstance(data); - + //if grain implements IStreamSubscriptionObserver, then install stream consumer extension on it if(grain is IStreamSubscriptionObserver) InstallStreamConsumerExtension(data, grain as IStreamSubscriptionObserver); @@ -722,7 +722,7 @@ private void CreateGrainInstance(string grainTypeName, ActivationData data, stri grain.Data = data; data.SetGrainInstance(grain); } - + activations.IncrementGrainCounter(grainClassName); if (logger.IsEnabled(LogLevel.Debug)) logger.Debug("CreateGrainInstance {0}{1}", data.Grain, data.ActivationId); @@ -932,11 +932,11 @@ private void DestroyActivationAsync(ActivationData activation, MultiTaskCompleti /// // Overall code flow: // Deactivating state was already set before, in the correct context under lock. - // that means no more new requests will be accepted into this activation and all timer were stopped (no new ticks will be delivered or enqueued) + // that means no more new requests will be accepted into this activation and all timer were stopped (no new ticks will be delivered or enqueued) // Wait for all already scheduled ticks to finish // CallGrainDeactivate // when AsyncDeactivate promise is resolved (NOT when all Deactivate turns are done, which may be orphan tasks): - // Unregister in the directory + // Unregister in the directory // when all AsyncDeactivate turns are done (Dispatcher.OnActivationCompletedRequest): // Set Invalid state // UnregisterMessageTarget -> no new tasks will be enqueue (if an orphan task get enqueud, it is ignored and dropped on the floor). @@ -1003,7 +1003,7 @@ private async void FinishDestroyActivations(List list, int numbe //logger.Info(ErrorCode.Catalog_DestroyActivations_Done, "Starting FinishDestroyActivations #{0} - with {1} Activations.", number, list.Count); // step 3 - UnregisterManyAsync try - { + { List activationsToDeactivate = list. Where((ActivationData d) => d.IsUsingGrainDirectory). Select((ActivationData d) => ActivationAddress.GetAddress(LocalSilo, d.Grain, d.ActivationId)).ToList(); @@ -1039,7 +1039,7 @@ await scheduler.RunOrQueueTask(() => // IMPORTANT: no more awaits and .Ignore after that point. - // Just use this opportunity to invalidate local Cache Entry as well. + // Just use this opportunity to invalidate local Cache Entry as well. // If this silo is not the grain directory partition for this grain, it may have it in its cache. try { @@ -1188,7 +1188,7 @@ private async Task CallGrainDeactivateAndCleanupStreams(Activati if (TryGetActivationData(activation.ActivationId, out ignore) && activation.State == ActivationState.Deactivating) { - RequestContext.Clear(); // Clear any previous RC, so it does not leak into this call by mistake. + RequestContext.Clear(); // Clear any previous RC, so it does not leak into this call by mistake. await activation.Lifecycle.OnStop().WithCancellation(ct); } if (logger.IsEnabled(LogLevel.Debug)) logger.Debug(ErrorCode.Catalog_AfterCallingDeactivate, "Returned from calling {1} grain's OnDeactivateAsync() method {0}", activation, grainTypeName); @@ -1233,7 +1233,7 @@ private struct ActivationRegistrationResult /// public static readonly ActivationRegistrationResult Success = new ActivationRegistrationResult { - IsSuccess = true + IsSuccess = true }; public ActivationRegistrationResult(ActivationAddress existingActivationAddress) @@ -1242,7 +1242,7 @@ public ActivationRegistrationResult(ActivationAddress existingActivationAddress) ExistingActivationAddress = existingActivationAddress; IsSuccess = false; } - + /// /// Returns true if this instance represents a successful registration, false otherwise. /// @@ -1264,13 +1264,13 @@ private async Task RegisterActivationInGrainDirect { ActivationAddress address = activation.Address; - // Currently, the only grain type that is not registered in the Grain Directory is StatelessWorker. + // Currently, the only grain type that is not registered in the Grain Directory is StatelessWorker. // Among those that are registered in the directory, we currently do not have any multi activations. if (activation.IsUsingGrainDirectory) { var result = await scheduler.RunOrQueueTask(() => directory.RegisterAsync(address, singleActivation:true), this.SchedulingContext); if (address.Equals(result.Address)) return ActivationRegistrationResult.Success; - + return new ActivationRegistrationResult(existingActivationAddress: result.Address); } else if (activation.PlacedUsing is StatelessWorkerPlacement stPlacement) @@ -1302,7 +1302,7 @@ private async Task RegisterActivationInGrainDirect } } - // We currently don't have any other case for multiple activations except for StatelessWorker. + // We currently don't have any other case for multiple activations except for StatelessWorker. } /// @@ -1388,7 +1388,7 @@ private List TryGetActivationDatas(List addre } private void OnSiloStatusChange(SiloAddress updatedSilo, SiloStatus status) - { + { // ignore joining events and also events on myself. if (updatedSilo.Equals(LocalSilo)) return; diff --git a/src/Orleans.Runtime/Core/Dispatcher.cs b/src/Orleans.Runtime/Core/Dispatcher.cs index d12ce865678..0682ca29a38 100644 --- a/src/Orleans.Runtime/Core/Dispatcher.cs +++ b/src/Orleans.Runtime/Core/Dispatcher.cs @@ -276,7 +276,7 @@ private void ReceiveRequest(Message message, ActivationData targetActivation) { try { - CheckDeadlock(message); + CheckDeadlock(targetActivation, message); } catch (DeadlockException exc) { @@ -380,11 +380,12 @@ private bool IsMessageACallChainLoop(Message message) var prevChain = ((IList)obj); ActivationId nextActivationId = message.TargetActivation; + // check if the target activation already appears in the call chain. foreach (object invocationObj in prevChain) { var prevId = ((RequestInvocationHistorySummary)invocationObj).ActivationId; - if (prevId.Equals(nextActivationId) && !catalog.CanInterleave(nextActivationId, message)) + if (prevId.Equals(nextActivationId)) { return true; } @@ -397,10 +398,11 @@ private bool IsMessageACallChainLoop(Message message) /// Check if the current message will cause deadlock. /// Throw DeadlockException if yes. /// + /// /// Message to analyze - private void CheckDeadlock(Message message) + private void CheckDeadlock(ActivationData targetActivation, Message message) { - if (IsMessageACallChainLoop(message)) + if (IsMessageACallChainLoop(message) && !catalog.CanInterleave(targetActivation.ActivationId, message)) { IEnumerable> callChain = Enumerable.Empty>(); string exceptionMessage = string.Empty; From 66a6f5156e7a9e20991e8c0999dec7a4808200aa Mon Sep 17 00:00:00 2001 From: Andrew Hesky Date: Mon, 29 Oct 2018 12:51:59 -0700 Subject: [PATCH 05/10] back to expected values for deadlock detection and numIterations in test --- .../AllowCallChainReentrancyEntireChainTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/TesterInternal/AllowCallChainReentrancyEntireChainTests.cs b/test/TesterInternal/AllowCallChainReentrancyEntireChainTests.cs index 4260e7b988b..67c8496d205 100644 --- a/test/TesterInternal/AllowCallChainReentrancyEntireChainTests.cs +++ b/test/TesterInternal/AllowCallChainReentrancyEntireChainTests.cs @@ -30,7 +30,7 @@ public void Configure(ISiloHostBuilder hostBuilder) { hostBuilder.Configure(options => { - options.PerformDeadlockDetection = true; + options.PerformDeadlockDetection = false; options.CallChainReentrancy = SchedulingOptions.CallChainReentrancyStrategy.EntireChain; }).ConfigureLogging(logging => { @@ -41,7 +41,7 @@ public void Configure(ISiloHostBuilder hostBuilder) } } - private const int numIterations = 1; + private const int numIterations = 30; private readonly CallChainReentrancyTestHelper testHelper; From f92470dfdb3482b9b1325e691a0398c470f3380c Mon Sep 17 00:00:00 2001 From: Andrew Hesky Date: Tue, 13 Nov 2018 16:18:09 -0800 Subject: [PATCH 06/10] rollback bool to enum change --- .../LegacyClusterConfigurationExtensions.cs | 5 +--- .../Options/SchedulingOptions.cs | 26 ++----------------- src/Orleans.Runtime/Core/Dispatcher.cs | 19 ++------------ .../Core/InsideRuntimeClient.cs | 3 +-- ...ts.cs => AllowCallChainReentrancyTests.cs} | 6 ++--- ...AllowCallChainSingleCallReentrancyTests.cs | 9 ++++++- ...ionWithoutAllowCallChainReentrancyTests.cs | 2 +- 7 files changed, 18 insertions(+), 52 deletions(-) rename test/TesterInternal/{AllowCallChainReentrancyEntireChainTests.cs => AllowCallChainReentrancyTests.cs} (92%) diff --git a/src/Orleans.Runtime.Legacy/Configuration/LegacyClusterConfigurationExtensions.cs b/src/Orleans.Runtime.Legacy/Configuration/LegacyClusterConfigurationExtensions.cs index 96656c00360..18ec2c7c792 100644 --- a/src/Orleans.Runtime.Legacy/Configuration/LegacyClusterConfigurationExtensions.cs +++ b/src/Orleans.Runtime.Legacy/Configuration/LegacyClusterConfigurationExtensions.cs @@ -189,10 +189,7 @@ private static void AddLegacyClusterConfigurationSupport(IServiceCollection serv services.AddOptions() .Configure((options, config) => { - options.CallChainReentrancy = config.AllowCallChainReentrancy ? - SchedulingOptions.CallChainReentrancyStrategy.SingleCall - : SchedulingOptions.CallChainReentrancyStrategy.None; - + options.AllowCallChainReentrancy = config.AllowCallChainReentrancy; options.PerformDeadlockDetection = config.PerformDeadlockDetection; }) .Configure((options, nodeConfig) => diff --git a/src/Orleans.Runtime/Configuration/Options/SchedulingOptions.cs b/src/Orleans.Runtime/Configuration/Options/SchedulingOptions.cs index e0d4749f377..01ec5909802 100644 --- a/src/Orleans.Runtime/Configuration/Options/SchedulingOptions.cs +++ b/src/Orleans.Runtime/Configuration/Options/SchedulingOptions.cs @@ -9,19 +9,6 @@ namespace Orleans.Configuration /// public class SchedulingOptions { - /// - /// Configuration type that controls the type of the grain directory caching algorithm that silo use. - /// - public enum CallChainReentrancyStrategy - { - /// Don't allow reentrancy. - None, - /// Allow reentrancy for a single call (A->B->A). - SingleCall, - /// Allow reentrancy for for entire call chain (A->B->C->A). - EntireChain - } - /// /// Whether or not to perform deadlock detection. /// @@ -31,17 +18,8 @@ public enum CallChainReentrancyStrategy /// /// Whether or not to allow reentrancy for calls within the same call chain. /// - [Obsolete("AllowCallChainReentrancy is obsolete, use CallChainReentrancy. None is equivalent to false, SingleCall is equivalent to true")] - public bool AllowCallChainReentrancy { - get => CallChainReentrancy != CallChainReentrancyStrategy.None; - set => CallChainReentrancy = value ? CallChainReentrancyStrategy.SingleCall : CallChainReentrancyStrategy.None; - } - - /// - /// How to handle reentrancy for calls within the same call chain. - /// - public CallChainReentrancyStrategy CallChainReentrancy { get; set; } = DEFAULT_CALL_CHAIN_REENTRANCY; - public const CallChainReentrancyStrategy DEFAULT_CALL_CHAIN_REENTRANCY = CallChainReentrancyStrategy.SingleCall; + public bool AllowCallChainReentrancy { get; set; } = DEFAULT_ALLOW_CALL_CHAIN_REENTRANCY; + public const bool DEFAULT_ALLOW_CALL_CHAIN_REENTRANCY = true; /// /// The MaxActiveThreads attribute specifies the maximum number of simultaneous active threads the scheduler will allow. diff --git a/src/Orleans.Runtime/Core/Dispatcher.cs b/src/Orleans.Runtime/Core/Dispatcher.cs index 0682ca29a38..18942134787 100644 --- a/src/Orleans.Runtime/Core/Dispatcher.cs +++ b/src/Orleans.Runtime/Core/Dispatcher.cs @@ -327,9 +327,7 @@ public bool CanInterleave(ActivationData targetActivation, Message incoming) incoming.IsAlwaysInterleave || targetActivation.Running == null || targetActivation.Running.IsReadOnly && incoming.IsReadOnly - || schedulingOptions.CallChainReentrancy == SchedulingOptions.CallChainReentrancyStrategy.SingleCall - && targetActivation.ActivationId.Equals(incoming.SendingActivation) // TODO, consider using the same method for both modes - || schedulingOptions.CallChainReentrancy == SchedulingOptions.CallChainReentrancyStrategy.EntireChain + || schedulingOptions.AllowCallChainReentrancy && IsMessageACallChainLoop(incoming) || catalog.CanInterleave(targetActivation.ActivationId, incoming); @@ -346,20 +344,7 @@ public bool CanInterleave(ActivationData targetActivation, Message incoming) /// private void MarkSameCallChainMessageAsInterleaving(ActivationData sendingActivation, Message outgoing) { - if (schedulingOptions.CallChainReentrancy == SchedulingOptions.CallChainReentrancyStrategy.None) - { - return; - } - - // TODO consider using the same method for both strategies - if (schedulingOptions.CallChainReentrancy == SchedulingOptions.CallChainReentrancyStrategy.SingleCall - && sendingActivation?.RunningRequestsSenders.Contains(outgoing.TargetActivation) == true) - { - outgoing.IsAlwaysInterleave = true; - } - - if (schedulingOptions.CallChainReentrancy == SchedulingOptions.CallChainReentrancyStrategy.EntireChain - && IsMessageACallChainLoop(outgoing)) + if (!schedulingOptions.AllowCallChainReentrancy && IsMessageACallChainLoop(outgoing)) { outgoing.IsAlwaysInterleave = true; } diff --git a/src/Orleans.Runtime/Core/InsideRuntimeClient.cs b/src/Orleans.Runtime/Core/InsideRuntimeClient.cs index 2f60d260f6a..726b65c9b65 100644 --- a/src/Orleans.Runtime/Core/InsideRuntimeClient.cs +++ b/src/Orleans.Runtime/Core/InsideRuntimeClient.cs @@ -289,8 +289,7 @@ public async Task Invoke(IAddressable target, IInvokable invokable, Message mess } RequestContextExtensions.Import(message.RequestContextData); - if ((schedulingOptions.PerformDeadlockDetection - || schedulingOptions.CallChainReentrancy == SchedulingOptions.CallChainReentrancyStrategy.EntireChain) + if ((schedulingOptions.PerformDeadlockDetection || schedulingOptions.AllowCallChainReentrancy) && !message.TargetGrain.IsSystemTarget) { RequestInvocationHistorySummary invocationHistory = !schedulingOptions.PerformDeadlockDetection ? diff --git a/test/TesterInternal/AllowCallChainReentrancyEntireChainTests.cs b/test/TesterInternal/AllowCallChainReentrancyTests.cs similarity index 92% rename from test/TesterInternal/AllowCallChainReentrancyEntireChainTests.cs rename to test/TesterInternal/AllowCallChainReentrancyTests.cs index 67c8496d205..dbf2e491fb2 100644 --- a/test/TesterInternal/AllowCallChainReentrancyEntireChainTests.cs +++ b/test/TesterInternal/AllowCallChainReentrancyTests.cs @@ -10,7 +10,7 @@ namespace UnitTests.General { - public class AllowCallChainReentrancyEntireChainTests : OrleansTestingBase + public class AllowCallChainReentrancyTests : OrleansTestingBase { private readonly Fixture fixture; @@ -31,7 +31,7 @@ public void Configure(ISiloHostBuilder hostBuilder) hostBuilder.Configure(options => { options.PerformDeadlockDetection = false; - options.CallChainReentrancy = SchedulingOptions.CallChainReentrancyStrategy.EntireChain; + options.AllowCallChainReentrancy = true; }).ConfigureLogging(logging => { logging.AddProvider(new TestLoggingProvider(x => _output.WriteLine(x))); @@ -45,7 +45,7 @@ public void Configure(ISiloHostBuilder hostBuilder) private readonly CallChainReentrancyTestHelper testHelper; - public AllowCallChainReentrancyEntireChainTests(ITestOutputHelper output) + public AllowCallChainReentrancyTests(ITestOutputHelper output) { if(output == null) throw new ArgumentNullException(nameof(output)); diff --git a/test/TesterInternal/DeadlockDetectionWithAllowCallChainSingleCallReentrancyTests.cs b/test/TesterInternal/DeadlockDetectionWithAllowCallChainSingleCallReentrancyTests.cs index cc19b96d7f9..3242967e81f 100644 --- a/test/TesterInternal/DeadlockDetectionWithAllowCallChainSingleCallReentrancyTests.cs +++ b/test/TesterInternal/DeadlockDetectionWithAllowCallChainSingleCallReentrancyTests.cs @@ -29,7 +29,7 @@ public void Configure(ISiloHostBuilder hostBuilder) hostBuilder.Configure(options => { options.PerformDeadlockDetection = true; - options.CallChainReentrancy = SchedulingOptions.CallChainReentrancyStrategy.SingleCall; + options.AllowCallChainReentrancy = true; }); } } @@ -86,5 +86,12 @@ public async Task DeadlockDetection_5() { await testHelper.DeadlockDetection_5(); } + + // 6) Allowed reentrancy on non-reentrant grains A, B, C, A + [Fact, TestCategory("Functional"), TestCategory("Deadlock")] + public async Task DeadlockDetection_6() + { + await testHelper.DeadlockDetection_6(); + } } } diff --git a/test/TesterInternal/DeadlockDetectionWithoutAllowCallChainReentrancyTests.cs b/test/TesterInternal/DeadlockDetectionWithoutAllowCallChainReentrancyTests.cs index 77fa4284b0c..3a92bbd4035 100644 --- a/test/TesterInternal/DeadlockDetectionWithoutAllowCallChainReentrancyTests.cs +++ b/test/TesterInternal/DeadlockDetectionWithoutAllowCallChainReentrancyTests.cs @@ -30,7 +30,7 @@ public void Configure(ISiloHostBuilder hostBuilder) hostBuilder.Configure(options => { options.PerformDeadlockDetection = true; - options.CallChainReentrancy = SchedulingOptions.CallChainReentrancyStrategy.None; + options.AllowCallChainReentrancy = false; }); } } From 1918ffbdac2bbaa2034c4194cd71fbb57701d568 Mon Sep 17 00:00:00 2001 From: Andrew Hesky Date: Tue, 13 Nov 2018 17:13:30 -0800 Subject: [PATCH 07/10] bugfix and cleanup --- .../OptionLogger/DefaultOptionsFormatter.cs | 1 - src/Orleans.Runtime/Catalog/ActivationData.cs | 42 ++++++++----------- src/Orleans.Runtime/Core/Dispatcher.cs | 2 +- .../Core/InsideRuntimeClient.cs | 17 +++++--- 4 files changed, 29 insertions(+), 33 deletions(-) diff --git a/src/Orleans.Core/Configuration/OptionLogger/DefaultOptionsFormatter.cs b/src/Orleans.Core/Configuration/OptionLogger/DefaultOptionsFormatter.cs index 31cd74adaef..f7fbe4d0748 100644 --- a/src/Orleans.Core/Configuration/OptionLogger/DefaultOptionsFormatter.cs +++ b/src/Orleans.Core/Configuration/OptionLogger/DefaultOptionsFormatter.cs @@ -3,7 +3,6 @@ using System.Collections.Generic; using System.Linq; using System.Reflection; -using System.Security.Cryptography.X509Certificates; namespace Orleans { diff --git a/src/Orleans.Runtime/Catalog/ActivationData.cs b/src/Orleans.Runtime/Catalog/ActivationData.cs index a5bc74b4c61..d4de86b018e 100644 --- a/src/Orleans.Runtime/Catalog/ActivationData.cs +++ b/src/Orleans.Runtime/Catalog/ActivationData.cs @@ -24,7 +24,7 @@ namespace Orleans.Runtime /// internal class ActivationData : IGrainActivationContext, IActivationData, IInvokable, IDisposable { - // This class is used for activations that have extension invokers. It keeps a dictionary of + // This class is used for activations that have extension invokers. It keeps a dictionary of // invoker objects to use with the activation, and extend the default invoker // defined for the grain class. // Note that in all cases we never have more than one copy of an actual invoker; @@ -34,7 +34,7 @@ private class ExtensionInvoker : IGrainMethodInvoker, IGrainExtensionMap // Because calls to ExtensionInvoker are allways made within the activation context, // we rely on the single-threading guarantee of the runtime and do not protect the map with a lock. private Dictionary> extensionMap; // key is the extension interface ID - + /// /// Try to add an extension for the specific interface ID. /// Fail and return false if there is already an extension for that interface ID. @@ -171,7 +171,7 @@ internal class GrainActivationContextFactory private IGrainMethodInvoker lastInvoker; private IServiceScope serviceScope; private HashSet timers; - + public ActivationData( ActivationAddress addr, string genericArguments, @@ -264,8 +264,6 @@ internal bool TryGetExtensionHandler(Type extensionType, out IGrainExtension res return extensionInvoker != null && extensionInvoker.TryGetExtensionHandler(extensionType, out result); } - public HashSet RunningRequestsSenders { get; } = new HashSet(); - public ISchedulingContext SchedulingContext { get; } public string GrainTypeName @@ -313,14 +311,14 @@ private static void SetGrainActivationContextInScopedServices(IServiceProvider s var contextFactory = sp.GetRequiredService(); contextFactory.Context = context; } - + private Streams.StreamDirectory streamDirectory; internal Streams.StreamDirectory GetStreamDirectory() { return streamDirectory ?? (streamDirectory = new Streams.StreamDirectory()); } - internal bool IsUsingStreams + internal bool IsUsingStreams { get { return streamDirectory != null; } } @@ -343,7 +341,7 @@ GrainReference IActivationData.GrainReference { get { return GrainReference; } } - + public GrainId Identity { get { return Grain; } @@ -435,7 +433,7 @@ public void ResetCollectionTicket() public void SetCollectionTicket(DateTime ticket) { if (ticket == default(DateTime)) throw new ArgumentException("default(DateTime) is disallowed", "ticket"); - if (CollectionTicket != default(DateTime)) + if (CollectionTicket != default(DateTime)) { throw new InvalidOperationException("call ResetCollectionTicket before calling SetCollectionTicket."); } @@ -449,7 +447,7 @@ public void SetCollectionTicket(DateTime ticket) // Currently, the only supported multi-activation grain is one using the StatelessWorkerPlacement strategy. internal bool IsStatelessWorker => this.PlacedUsing is StatelessWorkerPlacement; - + /// /// Returns a value indicating whether or not this placement strategy requires activations to be registered in /// the grain directory. @@ -471,12 +469,6 @@ public void RecordRunning(Message message) // Note: This method is always called while holding lock on this activation, so no need for additional locks here numRunning++; - if (message.Direction != Message.Directions.OneWay - && message.SendingActivation != null - && !message.SendingGrain?.IsClient == true) - { - RunningRequestsSenders.Add(message.SendingActivation); - } if (Running != null) return; @@ -490,7 +482,7 @@ public void ResetRunning(Message message) { // Note: This method is always called while holding lock on this activation, so no need for additional locks here numRunning--; - RunningRequestsSenders.Remove(message.SendingActivation); + if (numRunning == 0) { becameIdle = DateTime.UtcNow; @@ -523,7 +515,7 @@ public void ResetRunning(Message message) /// Increment the number of in-flight messages currently being processed. public void IncrementInFlightCount() { Interlocked.Increment(ref inFlightCount); } - + /// Decrement the number of in-flight messages currently being processed. public void DecrementInFlightCount() { Interlocked.Decrement(ref inFlightCount); } @@ -532,14 +524,14 @@ public void ResetRunning(Message message) /// Decrement the number of messages currently in the process of being received. public void DecrementEnqueuedOnDispatcherCount() { Interlocked.Decrement(ref enqueuedOnDispatcherCount); } - + /// /// grouped by sending activation: responses first, then sorted by id /// private List waiting; - public int WaitingCount - { + public int WaitingCount + { get { return waiting == null ? 0 : waiting.Count; @@ -602,7 +594,7 @@ public EnqueueMessageResult EnqueueMessage(Message message) } /// - /// Check whether this activation is overloaded. + /// Check whether this activation is overloaded. /// Returns LimitExceededException if overloaded, otherwise nullc> /// /// Logger to use for reporting any overflow condition @@ -625,7 +617,7 @@ public LimitExceededException CheckOverloaded(ILogger log) if (maxRequestsHardLimit > 0 && count > maxRequestsHardLimit) // Hard limit { - log.Warn(ErrorCode.Catalog_Reject_ActivationTooManyRequests, + log.Warn(ErrorCode.Catalog_Reject_ActivationTooManyRequests, String.Format("Overload - {0} enqueued requests for activation {1}, exceeding hard limit rejection threshold of {2}", count, this, maxRequestsHardLimit)); @@ -699,7 +691,7 @@ public TimeSpan GetIdleness(DateTime now) { if (now == default(DateTime)) throw new ArgumentException("default(DateTime) is not allowed; Use DateTime.UtcNow instead.", "now"); - + return now - becameIdle; } @@ -803,7 +795,7 @@ public void OnTimerDisposed(IGrainTimer orleansTimerInsideGrain) internal Task WaitForAllTimersToFinish() { lock(this) - { + { if (timers == null) { return Task.CompletedTask; diff --git a/src/Orleans.Runtime/Core/Dispatcher.cs b/src/Orleans.Runtime/Core/Dispatcher.cs index 18942134787..358d232955e 100644 --- a/src/Orleans.Runtime/Core/Dispatcher.cs +++ b/src/Orleans.Runtime/Core/Dispatcher.cs @@ -344,7 +344,7 @@ public bool CanInterleave(ActivationData targetActivation, Message incoming) /// private void MarkSameCallChainMessageAsInterleaving(ActivationData sendingActivation, Message outgoing) { - if (!schedulingOptions.AllowCallChainReentrancy && IsMessageACallChainLoop(outgoing)) + if (schedulingOptions.AllowCallChainReentrancy && IsMessageACallChainLoop(outgoing)) { outgoing.IsAlwaysInterleave = true; } diff --git a/src/Orleans.Runtime/Core/InsideRuntimeClient.cs b/src/Orleans.Runtime/Core/InsideRuntimeClient.cs index 726b65c9b65..caf623909be 100644 --- a/src/Orleans.Runtime/Core/InsideRuntimeClient.cs +++ b/src/Orleans.Runtime/Core/InsideRuntimeClient.cs @@ -289,14 +289,19 @@ public async Task Invoke(IAddressable target, IInvokable invokable, Message mess } RequestContextExtensions.Import(message.RequestContextData); - if ((schedulingOptions.PerformDeadlockDetection || schedulingOptions.AllowCallChainReentrancy) - && !message.TargetGrain.IsSystemTarget) + + if (!message.TargetGrain.IsSystemTarget) { - RequestInvocationHistorySummary invocationHistory = !schedulingOptions.PerformDeadlockDetection ? - new RequestInvocationHistorySummary(message.TargetActivation) : - new RequestInvocationHistory(message.TargetGrain, message.TargetActivation, message.DebugContext); + if (schedulingOptions.PerformDeadlockDetection) + { + UpdateInvocationHistoryInRequestContext( + new RequestInvocationHistory(message.TargetGrain, message.TargetActivation, message.DebugContext)); + } + else if (schedulingOptions.AllowCallChainReentrancy) + { + UpdateInvocationHistoryInRequestContext(new RequestInvocationHistorySummary(message.TargetActivation)) + } - UpdateInvocationHistoryInRequestContext(invocationHistory); // RequestContext is automatically saved in the msg upon send and propagated to the next hop // in RuntimeClient.CreateMessage -> RequestContextExtensions.ExportToMessage(message); } From b1ea86f9cf27961ade5b009a91d379612f40240d Mon Sep 17 00:00:00 2001 From: Andrew Hesky Date: Wed, 14 Nov 2018 06:50:01 -0800 Subject: [PATCH 08/10] semicolon --- src/Orleans.Runtime/Core/InsideRuntimeClient.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Orleans.Runtime/Core/InsideRuntimeClient.cs b/src/Orleans.Runtime/Core/InsideRuntimeClient.cs index caf623909be..e0a906e3380 100644 --- a/src/Orleans.Runtime/Core/InsideRuntimeClient.cs +++ b/src/Orleans.Runtime/Core/InsideRuntimeClient.cs @@ -299,7 +299,8 @@ public async Task Invoke(IAddressable target, IInvokable invokable, Message mess } else if (schedulingOptions.AllowCallChainReentrancy) { - UpdateInvocationHistoryInRequestContext(new RequestInvocationHistorySummary(message.TargetActivation)) + UpdateInvocationHistoryInRequestContext( + new RequestInvocationHistorySummary(message.TargetActivation)); } // RequestContext is automatically saved in the msg upon send and propagated to the next hop From 9bd6101d7e0da7b803380de4d05d7a85604e1a8d Mon Sep 17 00:00:00 2001 From: Andrew Hesky Date: Wed, 14 Nov 2018 08:44:04 -0800 Subject: [PATCH 09/10] Don't try to dispose a null silo --- .../Streaming/HaloStreamSubscribeTests.cs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/test/Extensions/TesterAzureUtils/Streaming/HaloStreamSubscribeTests.cs b/test/Extensions/TesterAzureUtils/Streaming/HaloStreamSubscribeTests.cs index 7d359cd5d33..1791fb46795 100644 --- a/test/Extensions/TesterAzureUtils/Streaming/HaloStreamSubscribeTests.cs +++ b/test/Extensions/TesterAzureUtils/Streaming/HaloStreamSubscribeTests.cs @@ -75,12 +75,16 @@ public void Configure(ISiloHostBuilder hostBuilder) public override void Dispose() { base.Dispose(); - AzureQueueStreamProviderUtils.DeleteAllUsedAzureQueues(NullLoggerFactory.Instance, - AzureQueueUtilities.GenerateQueueNames(this.HostedCluster.Options.ClusterId, queueCount), - TestDefaultConfiguration.DataConnectionString).Wait(); - AzureQueueStreamProviderUtils.DeleteAllUsedAzureQueues(NullLoggerFactory.Instance, - AzureQueueUtilities.GenerateQueueNames($"{this.HostedCluster.Options.ClusterId}2", queueCount), - TestDefaultConfiguration.DataConnectionString).Wait(); + + if (HostedCluster != null) + { + AzureQueueStreamProviderUtils.DeleteAllUsedAzureQueues(NullLoggerFactory.Instance, + AzureQueueUtilities.GenerateQueueNames(this.HostedCluster.Options.ClusterId, queueCount), + TestDefaultConfiguration.DataConnectionString).Wait(); + AzureQueueStreamProviderUtils.DeleteAllUsedAzureQueues(NullLoggerFactory.Instance, + AzureQueueUtilities.GenerateQueueNames($"{this.HostedCluster.Options.ClusterId}2", queueCount), + TestDefaultConfiguration.DataConnectionString).Wait(); + } } } From c39e794e3789c89617002ca5acf9fe0d2138d266 Mon Sep 17 00:00:00 2001 From: Andrew Hesky Date: Wed, 14 Nov 2018 08:54:39 -0800 Subject: [PATCH 10/10] Remove hacky logging code --- test/TesterInternal/AllowCallChainReentrancyTests.cs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/test/TesterInternal/AllowCallChainReentrancyTests.cs b/test/TesterInternal/AllowCallChainReentrancyTests.cs index dbf2e491fb2..63d42c4fc67 100644 --- a/test/TesterInternal/AllowCallChainReentrancyTests.cs +++ b/test/TesterInternal/AllowCallChainReentrancyTests.cs @@ -16,9 +16,6 @@ public class AllowCallChainReentrancyTests : OrleansTestingBase public class Fixture : BaseTestClusterFixture { - // TODO static is really not what we want here - public static ITestOutputHelper _output { get; set; } - protected override void ConfigureTestCluster(TestClusterBuilder builder) { builder.AddSiloBuilderConfigurator(); @@ -32,9 +29,6 @@ public void Configure(ISiloHostBuilder hostBuilder) { options.PerformDeadlockDetection = false; options.AllowCallChainReentrancy = true; - }).ConfigureLogging(logging => - { - logging.AddProvider(new TestLoggingProvider(x => _output.WriteLine(x))); }); } @@ -49,7 +43,6 @@ public AllowCallChainReentrancyTests(ITestOutputHelper output) { if(output == null) throw new ArgumentNullException(nameof(output)); - Fixture._output = output; this.fixture = new Fixture(); testHelper = new CallChainReentrancyTestHelper {