From 7aee2c704d8af37421756a5d523fb3b63710be51 Mon Sep 17 00:00:00 2001 From: Ziya Suzen Date: Fri, 6 Mar 2026 13:48:46 +0000 Subject: [PATCH 01/19] Fix dispose bug on consumers --- .../ConsumerFetchTest.cs | 125 +++++++++++++++++- 1 file changed, 119 insertions(+), 6 deletions(-) diff --git a/tests/NATS.Client.JetStream.Tests/ConsumerFetchTest.cs b/tests/NATS.Client.JetStream.Tests/ConsumerFetchTest.cs index ed1ed755d..ece2ed43b 100644 --- a/tests/NATS.Client.JetStream.Tests/ConsumerFetchTest.cs +++ b/tests/NATS.Client.JetStream.Tests/ConsumerFetchTest.cs @@ -87,11 +87,18 @@ public async Task Fetch_dispose_test(NatsRequestReplyMode mode) await using var nats = new NatsConnection(new NatsOpts { Url = _server.Url, RequestReplyMode = mode }); var prefix = _server.GetNextId(); - var cts = new CancellationTokenSource(TimeSpan.FromSeconds(60)); + // var cts = new CancellationTokenSource(TimeSpan.FromSeconds(60)); + var cts = new CancellationTokenSource(); var js = new NatsJSContext(nats); - var stream = await js.CreateStreamAsync($"{prefix}s1", new[] { $"{prefix}s1.*" }, cts.Token); - var consumer = (NatsJSConsumer)await js.CreateOrUpdateConsumerAsync($"{prefix}s1", $"{prefix}c1", cancellationToken: cts.Token); + + var streamName = $"{prefix}s1"; + var subjectSub = $"{prefix}s1.*"; + var consumerName = $"{prefix}c1"; + var subject = $"{prefix}s1.foo"; + + await js.CreateStreamAsync(streamName, [subjectSub], cts.Token); + var consumer = (NatsJSConsumer)await js.CreateOrUpdateConsumerAsync(streamName, consumerName, cancellationToken: cts.Token); var fetchOpts = new NatsJSFetchOpts { @@ -102,7 +109,7 @@ public async Task Fetch_dispose_test(NatsRequestReplyMode mode) for (var i = 0; i < 100; i++) { - var ack = await js.PublishAsync($"{prefix}s1.foo", new TestData { Test = i }, serializer: TestDataJsonSerializer.Default, cancellationToken: cts.Token); + var ack = await js.PublishAsync(subject, new TestData { Test = i }, serializer: TestDataJsonSerializer.Default, cancellationToken: cts.Token); ack.EnsureSuccess(); } @@ -133,7 +140,7 @@ await Retry.Until( "ack pending 9", async () => { - var c = await js.GetConsumerAsync($"{prefix}s1", $"{prefix}c1", cts.Token); + var c = await js.GetConsumerAsync(streamName, consumerName, cts.Token); _output.WriteLine($"pend1:{c.Info.NumAckPending}"); return c.Info.NumAckPending == 9; }, @@ -150,7 +157,7 @@ await Retry.Until( "ack pending 0", async () => { - var c = await js.GetConsumerAsync($"{prefix}s1", $"{prefix}c1", cts.Token); + var c = await js.GetConsumerAsync(streamName, consumerName, cts.Token); _output.WriteLine($"pend:{c.Info.NumAckPending}"); return c.Info.NumAckPending == 0; }, @@ -159,4 +166,110 @@ await Retry.Until( await consumer.RefreshAsync(cts.Token); Assert.Equal(0, consumer.Info.NumAckPending); } + + [Fact] + public async Task Fetch_dispose_does_not_drop_buffered_messages() + { + // Pressure the race between _disposed=1 and TryComplete() by: + // 1. Restricting to a single CPU core (maximizes context switches) + // 2. Tight loop with large batches and immediate dispose + const int iterations = 50; + const int msgCount = 100; + + var proc = System.Diagnostics.Process.GetCurrentProcess(); + nint savedAffinity = 0; + try + { +#pragma warning disable CA1416 // Platform compatibility - wrapped in try/catch + savedAffinity = proc.ProcessorAffinity; + proc.ProcessorAffinity = (nint)1; +#pragma warning restore CA1416 + } + catch + { + // ProcessorAffinity may not be supported on all platforms + } + + try + { + await using var nats = new NatsConnection(new NatsOpts { Url = _server.Url }); + var prefix = _server.GetNextId(); + var cts = new CancellationTokenSource(TimeSpan.FromSeconds(120)); + + var js = new NatsJSContext(nats); + await js.CreateStreamAsync($"{prefix}s1", new[] { $"{prefix}s1.*" }, cts.Token); + + // Publish all messages once up front. + for (var i = 0; i < msgCount; i++) + { + var ack = await js.PublishAsync($"{prefix}s1.foo", new TestData { Test = i }, serializer: TestDataJsonSerializer.Default, cancellationToken: cts.Token); + ack.EnsureSuccess(); + } + + for (var iteration = 0; iteration < iterations; iteration++) + { + // Each iteration: new consumer, fetch, dispose fast, verify. + var consumer = (NatsJSConsumer)await js.CreateOrUpdateConsumerAsync( + $"{prefix}s1", + $"{prefix}c{iteration}", + cancellationToken: cts.Token); + + var fetchOpts = new NatsJSFetchOpts + { + MaxMsgs = msgCount, + IdleHeartbeat = TimeSpan.FromSeconds(5), + Expires = TimeSpan.FromSeconds(10), + }; + + var fc = await consumer.FetchInternalAsync( + serializer: TestDataJsonSerializer.Default, + opts: fetchOpts, + cancellationToken: cts.Token); + + // Wait for at least one message — proves messages are flowing. + await Retry.Until( + "first message in channel", + () => fc.Msgs.Count >= 1, + timeout: TimeSpan.FromSeconds(30)); + + // Dispose immediately while messages are still arriving. + await fc.DisposeAsync(); + + // Read and ack everything that made it into the channel. + var received = 0; + await foreach (var msg in fc.Msgs.ReadAllAsync(cts.Token)) + { + received++; + await msg.AckAsync(cancellationToken: cts.Token); + } + + // Verify: server should have zero ack-pending for delivered msgs. + // If messages were silently dropped, NumAckPending > 0. + await Retry.Until( + "ack pending 0", + async () => + { + await consumer.RefreshAsync(cts.Token); + _output.WriteLine($"iter:{iteration} received:{received} ackPending:{consumer.Info.NumAckPending}"); + return consumer.Info.NumAckPending == 0; + }, + retryDelay: TimeSpan.FromSeconds(1), + timeout: TimeSpan.FromSeconds(30)); + } + } + finally + { + try + { +#pragma warning disable CA1416 // Platform compatibility - wrapped in try/catch + if (savedAffinity != 0) + proc.ProcessorAffinity = savedAffinity; +#pragma warning restore CA1416 + } + catch + { + // Restore may fail if original wasn't supported + } + } + } } From 3cf70d6dff052ef3f26eeb9e71ff26986a6852dd Mon Sep 17 00:00:00 2001 From: Ziya Suzen Date: Wed, 11 Mar 2026 12:13:57 +0000 Subject: [PATCH 02/19] Improve `Fetch_dispose_does_not_drop_buffered_messages` test to ensure messages are not dropped during dispose. --- .../ConsumerFetchTest.cs | 189 +++++++++--------- 1 file changed, 100 insertions(+), 89 deletions(-) diff --git a/tests/NATS.Client.JetStream.Tests/ConsumerFetchTest.cs b/tests/NATS.Client.JetStream.Tests/ConsumerFetchTest.cs index ece2ed43b..f3adf802d 100644 --- a/tests/NATS.Client.JetStream.Tests/ConsumerFetchTest.cs +++ b/tests/NATS.Client.JetStream.Tests/ConsumerFetchTest.cs @@ -170,106 +170,117 @@ await Retry.Until( [Fact] public async Task Fetch_dispose_does_not_drop_buffered_messages() { - // Pressure the race between _disposed=1 and TryComplete() by: - // 1. Restricting to a single CPU core (maximizes context switches) - // 2. Tight loop with large batches and immediate dispose - const int iterations = 50; - const int msgCount = 100; - - var proc = System.Diagnostics.Process.GetCurrentProcess(); - nint savedAffinity = 0; - try - { -#pragma warning disable CA1416 // Platform compatibility - wrapped in try/catch - savedAffinity = proc.ProcessorAffinity; - proc.ProcessorAffinity = (nint)1; -#pragma warning restore CA1416 - } - catch - { - // ProcessorAffinity may not be supported on all platforms - } - - try - { - await using var nats = new NatsConnection(new NatsOpts { Url = _server.Url }); - var prefix = _server.GetNextId(); - var cts = new CancellationTokenSource(TimeSpan.FromSeconds(120)); + // Reproduce the race: messages delivered by server but dropped during + // dispose because the channel is completed before all writes finish. + // Strategy: pump thousands of messages so there are always messages + // in-flight during dispose, then check server stats for the gap. + // Run iterations in parallel (4x CPU count) to maximize contention. + var iterations = Environment.ProcessorCount * 4; + + await using var nats = new NatsConnection(new NatsOpts { Url = _server.Url }); + var prefix = _server.GetNextId(); + var cts = new CancellationTokenSource(TimeSpan.FromSeconds(120)); - var js = new NatsJSContext(nats); - await js.CreateStreamAsync($"{prefix}s1", new[] { $"{prefix}s1.*" }, cts.Token); + var js = new NatsJSContext(nats); + var streamName = $"{prefix}s1"; + var subject = $"{prefix}s1.foo"; - // Publish all messages once up front. - for (var i = 0; i < msgCount; i++) - { - var ack = await js.PublishAsync($"{prefix}s1.foo", new TestData { Test = i }, serializer: TestDataJsonSerializer.Default, cancellationToken: cts.Token); - ack.EnsureSuccess(); - } + await js.CreateStreamAsync(streamName, new[] { $"{prefix}s1.*" }, cts.Token); - for (var iteration = 0; iteration < iterations; iteration++) + // Keep publishing messages in the background throughout the test + // to ensure there are always messages in-flight during dispose. + var publishTask = Task.Run( + async () => { - // Each iteration: new consumer, fetch, dispose fast, verify. - var consumer = (NatsJSConsumer)await js.CreateOrUpdateConsumerAsync( - $"{prefix}s1", - $"{prefix}c{iteration}", - cancellationToken: cts.Token); - - var fetchOpts = new NatsJSFetchOpts + var seq = 0; + while (!cts.Token.IsCancellationRequested) { - MaxMsgs = msgCount, - IdleHeartbeat = TimeSpan.FromSeconds(5), - Expires = TimeSpan.FromSeconds(10), - }; - - var fc = await consumer.FetchInternalAsync( - serializer: TestDataJsonSerializer.Default, - opts: fetchOpts, - cancellationToken: cts.Token); - - // Wait for at least one message — proves messages are flowing. - await Retry.Until( - "first message in channel", - () => fc.Msgs.Count >= 1, - timeout: TimeSpan.FromSeconds(30)); - - // Dispose immediately while messages are still arriving. - await fc.DisposeAsync(); - - // Read and ack everything that made it into the channel. - var received = 0; - await foreach (var msg in fc.Msgs.ReadAllAsync(cts.Token)) - { - received++; - await msg.AckAsync(cancellationToken: cts.Token); + for (var i = 0; i < 10_000; i++) + { + var ack = await js.PublishAsync( + subject, + new TestData { Test = seq++ }, + serializer: TestDataJsonSerializer.Default, + cancellationToken: cts.Token); + ack.EnsureSuccess(); + } + + await Task.Delay(TimeSpan.FromMilliseconds(10), cts.Token).ConfigureAwait(false); } + }, + cts.Token); - // Verify: server should have zero ack-pending for delivered msgs. - // If messages were silently dropped, NumAckPending > 0. - await Retry.Until( - "ack pending 0", - async () => - { - await consumer.RefreshAsync(cts.Token); - _output.WriteLine($"iter:{iteration} received:{received} ackPending:{consumer.Info.NumAckPending}"); - return consumer.Info.NumAckPending == 0; - }, - retryDelay: TimeSpan.FromSeconds(1), - timeout: TimeSpan.FromSeconds(30)); - } - } - finally + // Wait for first batch to land before starting consumers. + await Retry.Until( + "initial messages published", + async () => + { + var stream = await js.GetStreamAsync(streamName, cancellationToken: cts.Token); + return stream.Info.State.Messages >= 10_000; + }, + timeout: TimeSpan.FromSeconds(30)); + + var tasks = Enumerable.Range(0, iterations).Select(iteration => Task.Run(async () => { - try + var consumerName = $"{prefix}c{iteration}"; + var consumer = (NatsJSConsumer)await js.CreateOrUpdateConsumerAsync( + streamName, + consumerName, + cancellationToken: cts.Token); + + var fetchOpts = new NatsJSFetchOpts { -#pragma warning disable CA1416 // Platform compatibility - wrapped in try/catch - if (savedAffinity != 0) - proc.ProcessorAffinity = savedAffinity; -#pragma warning restore CA1416 - } - catch + MaxMsgs = 100_000, + IdleHeartbeat = TimeSpan.FromSeconds(5), + Expires = TimeSpan.FromSeconds(30), + }; + + var fc = await consumer.FetchInternalAsync( + serializer: TestDataJsonSerializer.Default, + opts: fetchOpts, + cancellationToken: cts.Token); + + // Wait until some messages arrive but not all — we want + // messages still in-flight when we dispose. + await Retry.Until( + "some messages in channel", + () => fc.Msgs.Count >= 10, + timeout: TimeSpan.FromSeconds(30)); + + // Dispose while messages are still being delivered. + await fc.DisposeAsync(); + + // Read and ack everything that made it into the channel. + var received = 0; + await foreach (var msg in fc.Msgs.ReadAllAsync(cts.Token)) { - // Restore may fail if original wasn't supported + received++; + await msg.AckAsync(cancellationToken: cts.Token); } + + // Check server-side: delivered - acked should be zero. + // If messages were delivered but dropped before reaching the + // channel, NumAckPending > 0 (server sent them, we never acked). + await consumer.RefreshAsync(cts.Token); + var delivered = consumer.Info.Delivered.ConsumerSeq; + var ackFloor = consumer.Info.AckFloor.ConsumerSeq; + var ackPending = consumer.Info.NumAckPending; + + _output.WriteLine($"iter:{iteration} received:{received} delivered:{delivered} ackFloor:{ackFloor} ackPending:{ackPending}"); + + Assert.Equal(0, ackPending); + })).ToArray(); + + await Task.WhenAll(tasks); + + // Stop the background publisher. + cts.Cancel(); + try + { + await publishTask; + } + catch (OperationCanceledException) + { } } } From ac5c5af108c0f3a73f5c8080468d4ff83f489cf1 Mon Sep 17 00:00:00 2001 From: Ziya Suzen Date: Thu, 12 Mar 2026 11:50:26 +0000 Subject: [PATCH 03/19] Add `DrainAsync` to ensure safe disposal of subscriptions Streamlines the dispose logic by adding `DrainAsync` to handle UNSUB, PING/PONG, and channel completion to avoid message loss during shutdown. Applied changes to `NatsJSFetch` and `NatsJSConsume` for proper cleanup and consistent behavior. --- src/NATS.Client.Core/NatsSubBase.cs | 48 +++++++++++++++++++ .../Internal/NatsJSConsume.cs | 18 ++++--- .../Internal/NatsJSFetch.cs | 12 ++--- 3 files changed, 61 insertions(+), 17 deletions(-) diff --git a/src/NATS.Client.Core/NatsSubBase.cs b/src/NATS.Client.Core/NatsSubBase.cs index 72f3ec8c4..9dc02b01a 100644 --- a/src/NATS.Client.Core/NatsSubBase.cs +++ b/src/NATS.Client.Core/NatsSubBase.cs @@ -358,6 +358,54 @@ internal static bool IsHeader503(ReadOnlySequence? headersBuffer) => internal void ClearException() => Interlocked.Exchange(ref _exception, null); + /// + /// Sends UNSUB to the server, waits for PING/PONG round-trip to ensure + /// all in-flight messages are processed, then completes the channel. + /// This avoids the race where TryComplete() closes the channel while + /// the socket reader is still delivering messages. + /// + /// A that represents the asynchronous drain operation. + internal async ValueTask DrainAsync() + { + lock (_gate) + { + if (_unsubscribed) + return; + _unsubscribed = true; + } + + _timeoutTimer?.Change(Timeout.Infinite, Timeout.Infinite); + _idleTimeoutTimer?.Change(Timeout.Infinite, Timeout.Infinite); + _startUpTimeoutTimer?.Change(Timeout.Infinite, Timeout.Infinite); + + // Send UNSUB to the server and remove from subscription manager. + // This stops the server from sending new messages for this subscription. + await _manager.RemoveAsync(this).ConfigureAwait(false); + + // PING/PONG round-trip: after we get the PONG back, we know the server + // has processed our UNSUB and the socket reader has processed all messages + // that were in-flight before the UNSUB was received by the server. + try + { + using var pingCts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await Connection.PingAsync(pingCts.Token).ConfigureAwait(false); + } + catch (Exception e) + { + _logger.LogWarning(NatsLogEvents.Subscription, e, "Error during drain ping"); + } + + // Now it's safe to complete the channel — no more messages will arrive. + try + { + TryComplete(); + } + catch (Exception e) + { + _logger.LogWarning(NatsLogEvents.Subscription, e, "Error while completing subscription"); + } + } + /// /// Marks this subscription as a slow consumer. Returns true if this was a state transition /// (i.e., the subscription was not previously marked as a slow consumer). diff --git a/src/NATS.Client.JetStream/Internal/NatsJSConsume.cs b/src/NATS.Client.JetStream/Internal/NatsJSConsume.cs index dcc27cae5..0510553e5 100644 --- a/src/NATS.Client.JetStream/Internal/NatsJSConsume.cs +++ b/src/NATS.Client.JetStream/Internal/NatsJSConsume.cs @@ -211,6 +211,9 @@ public override async ValueTask DisposeAsync() Interlocked.Exchange(ref _disposed, 1); try { + // Drain: UNSUB → PING/PONG → TryComplete, so all in-flight + // messages are written to the channel before it's completed. + await DrainAsync().ConfigureAwait(false); await base.DisposeAsync().ConfigureAwait(false); } finally @@ -449,17 +452,12 @@ protected override async ValueTask ReceiveInternalAsync( NatsJSExtensionsInternal.TrySetPinIdFromHeaders(msg.Headers, _jsConsumer); - // Stop feeding the user if we are disposed. - // We need to exit as soon as possible. - if (Volatile.Read(ref _disposed) == 0) - { - // We can't pass cancellation token here because we need to hand - // the message to the user to be processed. Writer will be completed - // when the user calls Stop() or when the subscription is closed. - await _userMsgs.Writer.WriteAsync(msg).ConfigureAwait(false); + // We can't pass cancellation token here because we need to hand + // the message to the user to be processed. Writer will be completed + // when the user calls Stop() or when the subscription is closed. + await _userMsgs.Writer.WriteAsync(msg).ConfigureAwait(false); - ResetSlowConsumer(_userMsgs.Reader.Count); - } + ResetSlowConsumer(_userMsgs.Reader.Count); } } diff --git a/src/NATS.Client.JetStream/Internal/NatsJSFetch.cs b/src/NATS.Client.JetStream/Internal/NatsJSFetch.cs index f14dba6fc..22ca45b30 100644 --- a/src/NATS.Client.JetStream/Internal/NatsJSFetch.cs +++ b/src/NATS.Client.JetStream/Internal/NatsJSFetch.cs @@ -165,6 +165,9 @@ public override async ValueTask DisposeAsync() Interlocked.Exchange(ref _disposed, 1); try { + // Drain: UNSUB → PING/PONG → TryComplete, so all in-flight + // messages are written to the channel before it's completed. + await DrainAsync().ConfigureAwait(false); await base.DisposeAsync().ConfigureAwait(false); } finally @@ -294,14 +297,9 @@ protected override async ValueTask ReceiveInternalAsync( _pendingMsgs--; _pendingBytes -= msg.Size; - // Stop feeding the user if we are disposed. - // We need to exit as soon as possible. - if (Volatile.Read(ref _disposed) == 0) - { - await _userMsgs.Writer.WriteAsync(msg).ConfigureAwait(false); + await _userMsgs.Writer.WriteAsync(msg).ConfigureAwait(false); - ResetSlowConsumer(_userMsgs.Reader.Count); - } + ResetSlowConsumer(_userMsgs.Reader.Count); } if (_maxBytes > 0 && _pendingBytes <= 0) From 3c094ff11b19e430df7a8c82b44af2ca8b3b3e49 Mon Sep 17 00:00:00 2001 From: Ziya Suzen Date: Thu, 23 Apr 2026 13:56:42 +0100 Subject: [PATCH 04/19] Extend drain to ordered consumer, add repro sandbox Apply DrainAsync to NatsJSOrderedConsume so fetch, consume, and ordered-consume share the same UNSUB -> PING/PONG -> TryComplete dispose path. Replace the flaky in-test stress repro with a sandbox console app (SubCleanDispose) that exercises all four subscription types and checks for sequence gaps. --- NATS.Net.sln | 435 ++++++++++++++++++ sandbox/SubCleanDispose/Program.cs | 227 +++++++++ .../SubCleanDispose/SubCleanDispose.csproj | 16 + src/NATS.Client.Core/NatsSubBase.cs | 2 +- .../Internal/NatsJSConsume.cs | 2 +- .../Internal/NatsJSFetch.cs | 2 +- .../Internal/NatsJSOrderedConsume.cs | 18 +- .../ConsumerFetchTest.cs | 126 +---- 8 files changed, 696 insertions(+), 132 deletions(-) create mode 100644 sandbox/SubCleanDispose/Program.cs create mode 100644 sandbox/SubCleanDispose/SubCleanDispose.csproj diff --git a/NATS.Net.sln b/NATS.Net.sln index 0cd496046..4703b5ab5 100644 --- a/NATS.Net.sln +++ b/NATS.Net.sln @@ -158,220 +158,654 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NATS.Client.CheckAbiTransie EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NATS.Client.CheckAbi", "tests\NATS.Client.CheckAbi\NATS.Client.CheckAbi.csproj", "{DA8E9EC2-990C-4DA2-A3F3-7E7796C71DD8}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SubCleanDispose", "sandbox\SubCleanDispose\SubCleanDispose.csproj", "{B54702BF-ECAD-4F96-BC43-F5B670F0A668}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 Release|Any CPU = Release|Any CPU + Release|x64 = Release|x64 + Release|x86 = Release|x86 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution {702B4B02-7FEC-42A1-A399-6F39D4E2CA29}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {702B4B02-7FEC-42A1-A399-6F39D4E2CA29}.Debug|Any CPU.Build.0 = Debug|Any CPU + {702B4B02-7FEC-42A1-A399-6F39D4E2CA29}.Debug|x64.ActiveCfg = Debug|Any CPU + {702B4B02-7FEC-42A1-A399-6F39D4E2CA29}.Debug|x64.Build.0 = Debug|Any CPU + {702B4B02-7FEC-42A1-A399-6F39D4E2CA29}.Debug|x86.ActiveCfg = Debug|Any CPU + {702B4B02-7FEC-42A1-A399-6F39D4E2CA29}.Debug|x86.Build.0 = Debug|Any CPU {702B4B02-7FEC-42A1-A399-6F39D4E2CA29}.Release|Any CPU.ActiveCfg = Release|Any CPU {702B4B02-7FEC-42A1-A399-6F39D4E2CA29}.Release|Any CPU.Build.0 = Release|Any CPU + {702B4B02-7FEC-42A1-A399-6F39D4E2CA29}.Release|x64.ActiveCfg = Release|Any CPU + {702B4B02-7FEC-42A1-A399-6F39D4E2CA29}.Release|x64.Build.0 = Release|Any CPU + {702B4B02-7FEC-42A1-A399-6F39D4E2CA29}.Release|x86.ActiveCfg = Release|Any CPU + {702B4B02-7FEC-42A1-A399-6F39D4E2CA29}.Release|x86.Build.0 = Release|Any CPU {7FE6D43B-181E-485E-8C14-35EAA0EBF09A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {7FE6D43B-181E-485E-8C14-35EAA0EBF09A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7FE6D43B-181E-485E-8C14-35EAA0EBF09A}.Debug|x64.ActiveCfg = Debug|Any CPU + {7FE6D43B-181E-485E-8C14-35EAA0EBF09A}.Debug|x64.Build.0 = Debug|Any CPU + {7FE6D43B-181E-485E-8C14-35EAA0EBF09A}.Debug|x86.ActiveCfg = Debug|Any CPU + {7FE6D43B-181E-485E-8C14-35EAA0EBF09A}.Debug|x86.Build.0 = Debug|Any CPU {7FE6D43B-181E-485E-8C14-35EAA0EBF09A}.Release|Any CPU.ActiveCfg = Release|Any CPU {7FE6D43B-181E-485E-8C14-35EAA0EBF09A}.Release|Any CPU.Build.0 = Release|Any CPU + {7FE6D43B-181E-485E-8C14-35EAA0EBF09A}.Release|x64.ActiveCfg = Release|Any CPU + {7FE6D43B-181E-485E-8C14-35EAA0EBF09A}.Release|x64.Build.0 = Release|Any CPU + {7FE6D43B-181E-485E-8C14-35EAA0EBF09A}.Release|x86.ActiveCfg = Release|Any CPU + {7FE6D43B-181E-485E-8C14-35EAA0EBF09A}.Release|x86.Build.0 = Release|Any CPU {D60B2ADF-450D-4F8F-A3C5-7803D36FE06B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {D60B2ADF-450D-4F8F-A3C5-7803D36FE06B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D60B2ADF-450D-4F8F-A3C5-7803D36FE06B}.Debug|x64.ActiveCfg = Debug|Any CPU + {D60B2ADF-450D-4F8F-A3C5-7803D36FE06B}.Debug|x64.Build.0 = Debug|Any CPU + {D60B2ADF-450D-4F8F-A3C5-7803D36FE06B}.Debug|x86.ActiveCfg = Debug|Any CPU + {D60B2ADF-450D-4F8F-A3C5-7803D36FE06B}.Debug|x86.Build.0 = Debug|Any CPU {D60B2ADF-450D-4F8F-A3C5-7803D36FE06B}.Release|Any CPU.ActiveCfg = Release|Any CPU {D60B2ADF-450D-4F8F-A3C5-7803D36FE06B}.Release|Any CPU.Build.0 = Release|Any CPU + {D60B2ADF-450D-4F8F-A3C5-7803D36FE06B}.Release|x64.ActiveCfg = Release|Any CPU + {D60B2ADF-450D-4F8F-A3C5-7803D36FE06B}.Release|x64.Build.0 = Release|Any CPU + {D60B2ADF-450D-4F8F-A3C5-7803D36FE06B}.Release|x86.ActiveCfg = Release|Any CPU + {D60B2ADF-450D-4F8F-A3C5-7803D36FE06B}.Release|x86.Build.0 = Release|Any CPU {A10F0D89-13F3-49B3-ACF7-66E45DC95225}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {A10F0D89-13F3-49B3-ACF7-66E45DC95225}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A10F0D89-13F3-49B3-ACF7-66E45DC95225}.Debug|x64.ActiveCfg = Debug|Any CPU + {A10F0D89-13F3-49B3-ACF7-66E45DC95225}.Debug|x64.Build.0 = Debug|Any CPU + {A10F0D89-13F3-49B3-ACF7-66E45DC95225}.Debug|x86.ActiveCfg = Debug|Any CPU + {A10F0D89-13F3-49B3-ACF7-66E45DC95225}.Debug|x86.Build.0 = Debug|Any CPU {A10F0D89-13F3-49B3-ACF7-66E45DC95225}.Release|Any CPU.ActiveCfg = Release|Any CPU {A10F0D89-13F3-49B3-ACF7-66E45DC95225}.Release|Any CPU.Build.0 = Release|Any CPU + {A10F0D89-13F3-49B3-ACF7-66E45DC95225}.Release|x64.ActiveCfg = Release|Any CPU + {A10F0D89-13F3-49B3-ACF7-66E45DC95225}.Release|x64.Build.0 = Release|Any CPU + {A10F0D89-13F3-49B3-ACF7-66E45DC95225}.Release|x86.ActiveCfg = Release|Any CPU + {A10F0D89-13F3-49B3-ACF7-66E45DC95225}.Release|x86.Build.0 = Release|Any CPU {D3F09B30-1ED5-48C2-81CD-A2AD88E751AC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {D3F09B30-1ED5-48C2-81CD-A2AD88E751AC}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D3F09B30-1ED5-48C2-81CD-A2AD88E751AC}.Debug|x64.ActiveCfg = Debug|Any CPU + {D3F09B30-1ED5-48C2-81CD-A2AD88E751AC}.Debug|x64.Build.0 = Debug|Any CPU + {D3F09B30-1ED5-48C2-81CD-A2AD88E751AC}.Debug|x86.ActiveCfg = Debug|Any CPU + {D3F09B30-1ED5-48C2-81CD-A2AD88E751AC}.Debug|x86.Build.0 = Debug|Any CPU {D3F09B30-1ED5-48C2-81CD-A2AD88E751AC}.Release|Any CPU.ActiveCfg = Release|Any CPU {D3F09B30-1ED5-48C2-81CD-A2AD88E751AC}.Release|Any CPU.Build.0 = Release|Any CPU + {D3F09B30-1ED5-48C2-81CD-A2AD88E751AC}.Release|x64.ActiveCfg = Release|Any CPU + {D3F09B30-1ED5-48C2-81CD-A2AD88E751AC}.Release|x64.Build.0 = Release|Any CPU + {D3F09B30-1ED5-48C2-81CD-A2AD88E751AC}.Release|x86.ActiveCfg = Release|Any CPU + {D3F09B30-1ED5-48C2-81CD-A2AD88E751AC}.Release|x86.Build.0 = Release|Any CPU {44881DEE-8B49-44EA-B0BA-8BDA4F706E1A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {44881DEE-8B49-44EA-B0BA-8BDA4F706E1A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {44881DEE-8B49-44EA-B0BA-8BDA4F706E1A}.Debug|x64.ActiveCfg = Debug|Any CPU + {44881DEE-8B49-44EA-B0BA-8BDA4F706E1A}.Debug|x64.Build.0 = Debug|Any CPU + {44881DEE-8B49-44EA-B0BA-8BDA4F706E1A}.Debug|x86.ActiveCfg = Debug|Any CPU + {44881DEE-8B49-44EA-B0BA-8BDA4F706E1A}.Debug|x86.Build.0 = Debug|Any CPU {44881DEE-8B49-44EA-B0BA-8BDA4F706E1A}.Release|Any CPU.ActiveCfg = Release|Any CPU {44881DEE-8B49-44EA-B0BA-8BDA4F706E1A}.Release|Any CPU.Build.0 = Release|Any CPU + {44881DEE-8B49-44EA-B0BA-8BDA4F706E1A}.Release|x64.ActiveCfg = Release|Any CPU + {44881DEE-8B49-44EA-B0BA-8BDA4F706E1A}.Release|x64.Build.0 = Release|Any CPU + {44881DEE-8B49-44EA-B0BA-8BDA4F706E1A}.Release|x86.ActiveCfg = Release|Any CPU + {44881DEE-8B49-44EA-B0BA-8BDA4F706E1A}.Release|x86.Build.0 = Release|Any CPU {56A9B885-6B7E-4BC6-AED0-ADC67C9A68DB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {56A9B885-6B7E-4BC6-AED0-ADC67C9A68DB}.Debug|Any CPU.Build.0 = Debug|Any CPU + {56A9B885-6B7E-4BC6-AED0-ADC67C9A68DB}.Debug|x64.ActiveCfg = Debug|Any CPU + {56A9B885-6B7E-4BC6-AED0-ADC67C9A68DB}.Debug|x64.Build.0 = Debug|Any CPU + {56A9B885-6B7E-4BC6-AED0-ADC67C9A68DB}.Debug|x86.ActiveCfg = Debug|Any CPU + {56A9B885-6B7E-4BC6-AED0-ADC67C9A68DB}.Debug|x86.Build.0 = Debug|Any CPU {56A9B885-6B7E-4BC6-AED0-ADC67C9A68DB}.Release|Any CPU.ActiveCfg = Release|Any CPU {56A9B885-6B7E-4BC6-AED0-ADC67C9A68DB}.Release|Any CPU.Build.0 = Release|Any CPU + {56A9B885-6B7E-4BC6-AED0-ADC67C9A68DB}.Release|x64.ActiveCfg = Release|Any CPU + {56A9B885-6B7E-4BC6-AED0-ADC67C9A68DB}.Release|x64.Build.0 = Release|Any CPU + {56A9B885-6B7E-4BC6-AED0-ADC67C9A68DB}.Release|x86.ActiveCfg = Release|Any CPU + {56A9B885-6B7E-4BC6-AED0-ADC67C9A68DB}.Release|x86.Build.0 = Release|Any CPU {2D39F649-C512-4EE5-9DFA-7BD4D9E4F145}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {2D39F649-C512-4EE5-9DFA-7BD4D9E4F145}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2D39F649-C512-4EE5-9DFA-7BD4D9E4F145}.Debug|x64.ActiveCfg = Debug|Any CPU + {2D39F649-C512-4EE5-9DFA-7BD4D9E4F145}.Debug|x64.Build.0 = Debug|Any CPU + {2D39F649-C512-4EE5-9DFA-7BD4D9E4F145}.Debug|x86.ActiveCfg = Debug|Any CPU + {2D39F649-C512-4EE5-9DFA-7BD4D9E4F145}.Debug|x86.Build.0 = Debug|Any CPU {2D39F649-C512-4EE5-9DFA-7BD4D9E4F145}.Release|Any CPU.ActiveCfg = Release|Any CPU {2D39F649-C512-4EE5-9DFA-7BD4D9E4F145}.Release|Any CPU.Build.0 = Release|Any CPU + {2D39F649-C512-4EE5-9DFA-7BD4D9E4F145}.Release|x64.ActiveCfg = Release|Any CPU + {2D39F649-C512-4EE5-9DFA-7BD4D9E4F145}.Release|x64.Build.0 = Release|Any CPU + {2D39F649-C512-4EE5-9DFA-7BD4D9E4F145}.Release|x86.ActiveCfg = Release|Any CPU + {2D39F649-C512-4EE5-9DFA-7BD4D9E4F145}.Release|x86.Build.0 = Release|Any CPU {90E5BF38-70C1-460A-9177-CE42815BDBF5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {90E5BF38-70C1-460A-9177-CE42815BDBF5}.Debug|Any CPU.Build.0 = Debug|Any CPU + {90E5BF38-70C1-460A-9177-CE42815BDBF5}.Debug|x64.ActiveCfg = Debug|Any CPU + {90E5BF38-70C1-460A-9177-CE42815BDBF5}.Debug|x64.Build.0 = Debug|Any CPU + {90E5BF38-70C1-460A-9177-CE42815BDBF5}.Debug|x86.ActiveCfg = Debug|Any CPU + {90E5BF38-70C1-460A-9177-CE42815BDBF5}.Debug|x86.Build.0 = Debug|Any CPU {90E5BF38-70C1-460A-9177-CE42815BDBF5}.Release|Any CPU.ActiveCfg = Release|Any CPU {90E5BF38-70C1-460A-9177-CE42815BDBF5}.Release|Any CPU.Build.0 = Release|Any CPU + {90E5BF38-70C1-460A-9177-CE42815BDBF5}.Release|x64.ActiveCfg = Release|Any CPU + {90E5BF38-70C1-460A-9177-CE42815BDBF5}.Release|x64.Build.0 = Release|Any CPU + {90E5BF38-70C1-460A-9177-CE42815BDBF5}.Release|x86.ActiveCfg = Release|Any CPU + {90E5BF38-70C1-460A-9177-CE42815BDBF5}.Release|x86.Build.0 = Release|Any CPU {B7DD4A9C-2D24-4772-951E-86A665C59ADF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {B7DD4A9C-2D24-4772-951E-86A665C59ADF}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B7DD4A9C-2D24-4772-951E-86A665C59ADF}.Debug|x64.ActiveCfg = Debug|Any CPU + {B7DD4A9C-2D24-4772-951E-86A665C59ADF}.Debug|x64.Build.0 = Debug|Any CPU + {B7DD4A9C-2D24-4772-951E-86A665C59ADF}.Debug|x86.ActiveCfg = Debug|Any CPU + {B7DD4A9C-2D24-4772-951E-86A665C59ADF}.Debug|x86.Build.0 = Debug|Any CPU {B7DD4A9C-2D24-4772-951E-86A665C59ADF}.Release|Any CPU.ActiveCfg = Release|Any CPU {B7DD4A9C-2D24-4772-951E-86A665C59ADF}.Release|Any CPU.Build.0 = Release|Any CPU + {B7DD4A9C-2D24-4772-951E-86A665C59ADF}.Release|x64.ActiveCfg = Release|Any CPU + {B7DD4A9C-2D24-4772-951E-86A665C59ADF}.Release|x64.Build.0 = Release|Any CPU + {B7DD4A9C-2D24-4772-951E-86A665C59ADF}.Release|x86.ActiveCfg = Release|Any CPU + {B7DD4A9C-2D24-4772-951E-86A665C59ADF}.Release|x86.Build.0 = Release|Any CPU {ADF66CBA-4F3E-4E91-9842-E194E3BC06A1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {ADF66CBA-4F3E-4E91-9842-E194E3BC06A1}.Debug|Any CPU.Build.0 = Debug|Any CPU + {ADF66CBA-4F3E-4E91-9842-E194E3BC06A1}.Debug|x64.ActiveCfg = Debug|Any CPU + {ADF66CBA-4F3E-4E91-9842-E194E3BC06A1}.Debug|x64.Build.0 = Debug|Any CPU + {ADF66CBA-4F3E-4E91-9842-E194E3BC06A1}.Debug|x86.ActiveCfg = Debug|Any CPU + {ADF66CBA-4F3E-4E91-9842-E194E3BC06A1}.Debug|x86.Build.0 = Debug|Any CPU {ADF66CBA-4F3E-4E91-9842-E194E3BC06A1}.Release|Any CPU.ActiveCfg = Release|Any CPU {ADF66CBA-4F3E-4E91-9842-E194E3BC06A1}.Release|Any CPU.Build.0 = Release|Any CPU + {ADF66CBA-4F3E-4E91-9842-E194E3BC06A1}.Release|x64.ActiveCfg = Release|Any CPU + {ADF66CBA-4F3E-4E91-9842-E194E3BC06A1}.Release|x64.Build.0 = Release|Any CPU + {ADF66CBA-4F3E-4E91-9842-E194E3BC06A1}.Release|x86.ActiveCfg = Release|Any CPU + {ADF66CBA-4F3E-4E91-9842-E194E3BC06A1}.Release|x86.Build.0 = Release|Any CPU {A102AB7B-A90C-4717-B17C-045240838060}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {A102AB7B-A90C-4717-B17C-045240838060}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A102AB7B-A90C-4717-B17C-045240838060}.Debug|x64.ActiveCfg = Debug|Any CPU + {A102AB7B-A90C-4717-B17C-045240838060}.Debug|x64.Build.0 = Debug|Any CPU + {A102AB7B-A90C-4717-B17C-045240838060}.Debug|x86.ActiveCfg = Debug|Any CPU + {A102AB7B-A90C-4717-B17C-045240838060}.Debug|x86.Build.0 = Debug|Any CPU {A102AB7B-A90C-4717-B17C-045240838060}.Release|Any CPU.ActiveCfg = Release|Any CPU {A102AB7B-A90C-4717-B17C-045240838060}.Release|Any CPU.Build.0 = Release|Any CPU + {A102AB7B-A90C-4717-B17C-045240838060}.Release|x64.ActiveCfg = Release|Any CPU + {A102AB7B-A90C-4717-B17C-045240838060}.Release|x64.Build.0 = Release|Any CPU + {A102AB7B-A90C-4717-B17C-045240838060}.Release|x86.ActiveCfg = Release|Any CPU + {A102AB7B-A90C-4717-B17C-045240838060}.Release|x86.Build.0 = Release|Any CPU {908F2CED-CAC0-4A4E-AD19-362A413B5DA4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {908F2CED-CAC0-4A4E-AD19-362A413B5DA4}.Debug|Any CPU.Build.0 = Debug|Any CPU + {908F2CED-CAC0-4A4E-AD19-362A413B5DA4}.Debug|x64.ActiveCfg = Debug|Any CPU + {908F2CED-CAC0-4A4E-AD19-362A413B5DA4}.Debug|x64.Build.0 = Debug|Any CPU + {908F2CED-CAC0-4A4E-AD19-362A413B5DA4}.Debug|x86.ActiveCfg = Debug|Any CPU + {908F2CED-CAC0-4A4E-AD19-362A413B5DA4}.Debug|x86.Build.0 = Debug|Any CPU {908F2CED-CAC0-4A4E-AD19-362A413B5DA4}.Release|Any CPU.ActiveCfg = Release|Any CPU {908F2CED-CAC0-4A4E-AD19-362A413B5DA4}.Release|Any CPU.Build.0 = Release|Any CPU + {908F2CED-CAC0-4A4E-AD19-362A413B5DA4}.Release|x64.ActiveCfg = Release|Any CPU + {908F2CED-CAC0-4A4E-AD19-362A413B5DA4}.Release|x64.Build.0 = Release|Any CPU + {908F2CED-CAC0-4A4E-AD19-362A413B5DA4}.Release|x86.ActiveCfg = Release|Any CPU + {908F2CED-CAC0-4A4E-AD19-362A413B5DA4}.Release|x86.Build.0 = Release|Any CPU {3F8840BA-4F91-4359-AA53-6B26823E7F55}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {3F8840BA-4F91-4359-AA53-6B26823E7F55}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3F8840BA-4F91-4359-AA53-6B26823E7F55}.Debug|x64.ActiveCfg = Debug|Any CPU + {3F8840BA-4F91-4359-AA53-6B26823E7F55}.Debug|x64.Build.0 = Debug|Any CPU + {3F8840BA-4F91-4359-AA53-6B26823E7F55}.Debug|x86.ActiveCfg = Debug|Any CPU + {3F8840BA-4F91-4359-AA53-6B26823E7F55}.Debug|x86.Build.0 = Debug|Any CPU {3F8840BA-4F91-4359-AA53-6B26823E7F55}.Release|Any CPU.ActiveCfg = Release|Any CPU {3F8840BA-4F91-4359-AA53-6B26823E7F55}.Release|Any CPU.Build.0 = Release|Any CPU + {3F8840BA-4F91-4359-AA53-6B26823E7F55}.Release|x64.ActiveCfg = Release|Any CPU + {3F8840BA-4F91-4359-AA53-6B26823E7F55}.Release|x64.Build.0 = Release|Any CPU + {3F8840BA-4F91-4359-AA53-6B26823E7F55}.Release|x86.ActiveCfg = Release|Any CPU + {3F8840BA-4F91-4359-AA53-6B26823E7F55}.Release|x86.Build.0 = Release|Any CPU {BB2F4EEE-1AB3-43F7-B004-6C9B3D52353E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {BB2F4EEE-1AB3-43F7-B004-6C9B3D52353E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {BB2F4EEE-1AB3-43F7-B004-6C9B3D52353E}.Debug|x64.ActiveCfg = Debug|Any CPU + {BB2F4EEE-1AB3-43F7-B004-6C9B3D52353E}.Debug|x64.Build.0 = Debug|Any CPU + {BB2F4EEE-1AB3-43F7-B004-6C9B3D52353E}.Debug|x86.ActiveCfg = Debug|Any CPU + {BB2F4EEE-1AB3-43F7-B004-6C9B3D52353E}.Debug|x86.Build.0 = Debug|Any CPU {BB2F4EEE-1AB3-43F7-B004-6C9B3D52353E}.Release|Any CPU.ActiveCfg = Release|Any CPU {BB2F4EEE-1AB3-43F7-B004-6C9B3D52353E}.Release|Any CPU.Build.0 = Release|Any CPU + {BB2F4EEE-1AB3-43F7-B004-6C9B3D52353E}.Release|x64.ActiveCfg = Release|Any CPU + {BB2F4EEE-1AB3-43F7-B004-6C9B3D52353E}.Release|x64.Build.0 = Release|Any CPU + {BB2F4EEE-1AB3-43F7-B004-6C9B3D52353E}.Release|x86.ActiveCfg = Release|Any CPU + {BB2F4EEE-1AB3-43F7-B004-6C9B3D52353E}.Release|x86.Build.0 = Release|Any CPU {35578296-FF6C-4BA4-BEE5-C1A6E7DB7024}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {35578296-FF6C-4BA4-BEE5-C1A6E7DB7024}.Debug|Any CPU.Build.0 = Debug|Any CPU + {35578296-FF6C-4BA4-BEE5-C1A6E7DB7024}.Debug|x64.ActiveCfg = Debug|Any CPU + {35578296-FF6C-4BA4-BEE5-C1A6E7DB7024}.Debug|x64.Build.0 = Debug|Any CPU + {35578296-FF6C-4BA4-BEE5-C1A6E7DB7024}.Debug|x86.ActiveCfg = Debug|Any CPU + {35578296-FF6C-4BA4-BEE5-C1A6E7DB7024}.Debug|x86.Build.0 = Debug|Any CPU {35578296-FF6C-4BA4-BEE5-C1A6E7DB7024}.Release|Any CPU.ActiveCfg = Release|Any CPU {35578296-FF6C-4BA4-BEE5-C1A6E7DB7024}.Release|Any CPU.Build.0 = Release|Any CPU + {35578296-FF6C-4BA4-BEE5-C1A6E7DB7024}.Release|x64.ActiveCfg = Release|Any CPU + {35578296-FF6C-4BA4-BEE5-C1A6E7DB7024}.Release|x64.Build.0 = Release|Any CPU + {35578296-FF6C-4BA4-BEE5-C1A6E7DB7024}.Release|x86.ActiveCfg = Release|Any CPU + {35578296-FF6C-4BA4-BEE5-C1A6E7DB7024}.Release|x86.Build.0 = Release|Any CPU {050C63EE-8F1C-4535-9C6C-E12E62A1FF1D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {050C63EE-8F1C-4535-9C6C-E12E62A1FF1D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {050C63EE-8F1C-4535-9C6C-E12E62A1FF1D}.Debug|x64.ActiveCfg = Debug|Any CPU + {050C63EE-8F1C-4535-9C6C-E12E62A1FF1D}.Debug|x64.Build.0 = Debug|Any CPU + {050C63EE-8F1C-4535-9C6C-E12E62A1FF1D}.Debug|x86.ActiveCfg = Debug|Any CPU + {050C63EE-8F1C-4535-9C6C-E12E62A1FF1D}.Debug|x86.Build.0 = Debug|Any CPU {050C63EE-8F1C-4535-9C6C-E12E62A1FF1D}.Release|Any CPU.ActiveCfg = Release|Any CPU {050C63EE-8F1C-4535-9C6C-E12E62A1FF1D}.Release|Any CPU.Build.0 = Release|Any CPU + {050C63EE-8F1C-4535-9C6C-E12E62A1FF1D}.Release|x64.ActiveCfg = Release|Any CPU + {050C63EE-8F1C-4535-9C6C-E12E62A1FF1D}.Release|x64.Build.0 = Release|Any CPU + {050C63EE-8F1C-4535-9C6C-E12E62A1FF1D}.Release|x86.ActiveCfg = Release|Any CPU + {050C63EE-8F1C-4535-9C6C-E12E62A1FF1D}.Release|x86.Build.0 = Release|Any CPU {749CAE39-4C1E-4627-9E31-A36B987BC453}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {749CAE39-4C1E-4627-9E31-A36B987BC453}.Debug|Any CPU.Build.0 = Debug|Any CPU + {749CAE39-4C1E-4627-9E31-A36B987BC453}.Debug|x64.ActiveCfg = Debug|Any CPU + {749CAE39-4C1E-4627-9E31-A36B987BC453}.Debug|x64.Build.0 = Debug|Any CPU + {749CAE39-4C1E-4627-9E31-A36B987BC453}.Debug|x86.ActiveCfg = Debug|Any CPU + {749CAE39-4C1E-4627-9E31-A36B987BC453}.Debug|x86.Build.0 = Debug|Any CPU {749CAE39-4C1E-4627-9E31-A36B987BC453}.Release|Any CPU.ActiveCfg = Release|Any CPU {749CAE39-4C1E-4627-9E31-A36B987BC453}.Release|Any CPU.Build.0 = Release|Any CPU + {749CAE39-4C1E-4627-9E31-A36B987BC453}.Release|x64.ActiveCfg = Release|Any CPU + {749CAE39-4C1E-4627-9E31-A36B987BC453}.Release|x64.Build.0 = Release|Any CPU + {749CAE39-4C1E-4627-9E31-A36B987BC453}.Release|x86.ActiveCfg = Release|Any CPU + {749CAE39-4C1E-4627-9E31-A36B987BC453}.Release|x86.Build.0 = Release|Any CPU {CF44A42E-C075-4C5C-BE8B-3DF266FC617B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {CF44A42E-C075-4C5C-BE8B-3DF266FC617B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {CF44A42E-C075-4C5C-BE8B-3DF266FC617B}.Debug|x64.ActiveCfg = Debug|Any CPU + {CF44A42E-C075-4C5C-BE8B-3DF266FC617B}.Debug|x64.Build.0 = Debug|Any CPU + {CF44A42E-C075-4C5C-BE8B-3DF266FC617B}.Debug|x86.ActiveCfg = Debug|Any CPU + {CF44A42E-C075-4C5C-BE8B-3DF266FC617B}.Debug|x86.Build.0 = Debug|Any CPU {CF44A42E-C075-4C5C-BE8B-3DF266FC617B}.Release|Any CPU.ActiveCfg = Release|Any CPU {CF44A42E-C075-4C5C-BE8B-3DF266FC617B}.Release|Any CPU.Build.0 = Release|Any CPU + {CF44A42E-C075-4C5C-BE8B-3DF266FC617B}.Release|x64.ActiveCfg = Release|Any CPU + {CF44A42E-C075-4C5C-BE8B-3DF266FC617B}.Release|x64.Build.0 = Release|Any CPU + {CF44A42E-C075-4C5C-BE8B-3DF266FC617B}.Release|x86.ActiveCfg = Release|Any CPU + {CF44A42E-C075-4C5C-BE8B-3DF266FC617B}.Release|x86.Build.0 = Release|Any CPU {5B4B30A5-941B-44F9-98C6-06F0BB2242AB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {5B4B30A5-941B-44F9-98C6-06F0BB2242AB}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5B4B30A5-941B-44F9-98C6-06F0BB2242AB}.Debug|x64.ActiveCfg = Debug|Any CPU + {5B4B30A5-941B-44F9-98C6-06F0BB2242AB}.Debug|x64.Build.0 = Debug|Any CPU + {5B4B30A5-941B-44F9-98C6-06F0BB2242AB}.Debug|x86.ActiveCfg = Debug|Any CPU + {5B4B30A5-941B-44F9-98C6-06F0BB2242AB}.Debug|x86.Build.0 = Debug|Any CPU {5B4B30A5-941B-44F9-98C6-06F0BB2242AB}.Release|Any CPU.ActiveCfg = Release|Any CPU {5B4B30A5-941B-44F9-98C6-06F0BB2242AB}.Release|Any CPU.Build.0 = Release|Any CPU + {5B4B30A5-941B-44F9-98C6-06F0BB2242AB}.Release|x64.ActiveCfg = Release|Any CPU + {5B4B30A5-941B-44F9-98C6-06F0BB2242AB}.Release|x64.Build.0 = Release|Any CPU + {5B4B30A5-941B-44F9-98C6-06F0BB2242AB}.Release|x86.ActiveCfg = Release|Any CPU + {5B4B30A5-941B-44F9-98C6-06F0BB2242AB}.Release|x86.Build.0 = Release|Any CPU {6A7B9B9F-BFA4-4A6D-9006-0AAF597FC6DD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {6A7B9B9F-BFA4-4A6D-9006-0AAF597FC6DD}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6A7B9B9F-BFA4-4A6D-9006-0AAF597FC6DD}.Debug|x64.ActiveCfg = Debug|Any CPU + {6A7B9B9F-BFA4-4A6D-9006-0AAF597FC6DD}.Debug|x64.Build.0 = Debug|Any CPU + {6A7B9B9F-BFA4-4A6D-9006-0AAF597FC6DD}.Debug|x86.ActiveCfg = Debug|Any CPU + {6A7B9B9F-BFA4-4A6D-9006-0AAF597FC6DD}.Debug|x86.Build.0 = Debug|Any CPU {6A7B9B9F-BFA4-4A6D-9006-0AAF597FC6DD}.Release|Any CPU.ActiveCfg = Release|Any CPU {6A7B9B9F-BFA4-4A6D-9006-0AAF597FC6DD}.Release|Any CPU.Build.0 = Release|Any CPU + {6A7B9B9F-BFA4-4A6D-9006-0AAF597FC6DD}.Release|x64.ActiveCfg = Release|Any CPU + {6A7B9B9F-BFA4-4A6D-9006-0AAF597FC6DD}.Release|x64.Build.0 = Release|Any CPU + {6A7B9B9F-BFA4-4A6D-9006-0AAF597FC6DD}.Release|x86.ActiveCfg = Release|Any CPU + {6A7B9B9F-BFA4-4A6D-9006-0AAF597FC6DD}.Release|x86.Build.0 = Release|Any CPU {389C05EB-A0B3-4097-8C1F-4D55818438CC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {389C05EB-A0B3-4097-8C1F-4D55818438CC}.Debug|Any CPU.Build.0 = Debug|Any CPU + {389C05EB-A0B3-4097-8C1F-4D55818438CC}.Debug|x64.ActiveCfg = Debug|Any CPU + {389C05EB-A0B3-4097-8C1F-4D55818438CC}.Debug|x64.Build.0 = Debug|Any CPU + {389C05EB-A0B3-4097-8C1F-4D55818438CC}.Debug|x86.ActiveCfg = Debug|Any CPU + {389C05EB-A0B3-4097-8C1F-4D55818438CC}.Debug|x86.Build.0 = Debug|Any CPU {389C05EB-A0B3-4097-8C1F-4D55818438CC}.Release|Any CPU.ActiveCfg = Release|Any CPU {389C05EB-A0B3-4097-8C1F-4D55818438CC}.Release|Any CPU.Build.0 = Release|Any CPU + {389C05EB-A0B3-4097-8C1F-4D55818438CC}.Release|x64.ActiveCfg = Release|Any CPU + {389C05EB-A0B3-4097-8C1F-4D55818438CC}.Release|x64.Build.0 = Release|Any CPU + {389C05EB-A0B3-4097-8C1F-4D55818438CC}.Release|x86.ActiveCfg = Release|Any CPU + {389C05EB-A0B3-4097-8C1F-4D55818438CC}.Release|x86.Build.0 = Release|Any CPU {766C2486-34C3-4DD1-B31C-540C17C044B0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {766C2486-34C3-4DD1-B31C-540C17C044B0}.Debug|Any CPU.Build.0 = Debug|Any CPU + {766C2486-34C3-4DD1-B31C-540C17C044B0}.Debug|x64.ActiveCfg = Debug|Any CPU + {766C2486-34C3-4DD1-B31C-540C17C044B0}.Debug|x64.Build.0 = Debug|Any CPU + {766C2486-34C3-4DD1-B31C-540C17C044B0}.Debug|x86.ActiveCfg = Debug|Any CPU + {766C2486-34C3-4DD1-B31C-540C17C044B0}.Debug|x86.Build.0 = Debug|Any CPU {766C2486-34C3-4DD1-B31C-540C17C044B0}.Release|Any CPU.ActiveCfg = Release|Any CPU {766C2486-34C3-4DD1-B31C-540C17C044B0}.Release|Any CPU.Build.0 = Release|Any CPU + {766C2486-34C3-4DD1-B31C-540C17C044B0}.Release|x64.ActiveCfg = Release|Any CPU + {766C2486-34C3-4DD1-B31C-540C17C044B0}.Release|x64.Build.0 = Release|Any CPU + {766C2486-34C3-4DD1-B31C-540C17C044B0}.Release|x86.ActiveCfg = Release|Any CPU + {766C2486-34C3-4DD1-B31C-540C17C044B0}.Release|x86.Build.0 = Release|Any CPU {2EA0EB68-1AA4-40AC-8FA2-B51532075567}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {2EA0EB68-1AA4-40AC-8FA2-B51532075567}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2EA0EB68-1AA4-40AC-8FA2-B51532075567}.Debug|x64.ActiveCfg = Debug|Any CPU + {2EA0EB68-1AA4-40AC-8FA2-B51532075567}.Debug|x64.Build.0 = Debug|Any CPU + {2EA0EB68-1AA4-40AC-8FA2-B51532075567}.Debug|x86.ActiveCfg = Debug|Any CPU + {2EA0EB68-1AA4-40AC-8FA2-B51532075567}.Debug|x86.Build.0 = Debug|Any CPU {2EA0EB68-1AA4-40AC-8FA2-B51532075567}.Release|Any CPU.ActiveCfg = Release|Any CPU {2EA0EB68-1AA4-40AC-8FA2-B51532075567}.Release|Any CPU.Build.0 = Release|Any CPU + {2EA0EB68-1AA4-40AC-8FA2-B51532075567}.Release|x64.ActiveCfg = Release|Any CPU + {2EA0EB68-1AA4-40AC-8FA2-B51532075567}.Release|x64.Build.0 = Release|Any CPU + {2EA0EB68-1AA4-40AC-8FA2-B51532075567}.Release|x86.ActiveCfg = Release|Any CPU + {2EA0EB68-1AA4-40AC-8FA2-B51532075567}.Release|x86.Build.0 = Release|Any CPU {B8554582-DE19-41A2-9784-9B27C9F22429}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {B8554582-DE19-41A2-9784-9B27C9F22429}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B8554582-DE19-41A2-9784-9B27C9F22429}.Debug|x64.ActiveCfg = Debug|Any CPU + {B8554582-DE19-41A2-9784-9B27C9F22429}.Debug|x64.Build.0 = Debug|Any CPU + {B8554582-DE19-41A2-9784-9B27C9F22429}.Debug|x86.ActiveCfg = Debug|Any CPU + {B8554582-DE19-41A2-9784-9B27C9F22429}.Debug|x86.Build.0 = Debug|Any CPU {B8554582-DE19-41A2-9784-9B27C9F22429}.Release|Any CPU.ActiveCfg = Release|Any CPU {B8554582-DE19-41A2-9784-9B27C9F22429}.Release|Any CPU.Build.0 = Release|Any CPU + {B8554582-DE19-41A2-9784-9B27C9F22429}.Release|x64.ActiveCfg = Release|Any CPU + {B8554582-DE19-41A2-9784-9B27C9F22429}.Release|x64.Build.0 = Release|Any CPU + {B8554582-DE19-41A2-9784-9B27C9F22429}.Release|x86.ActiveCfg = Release|Any CPU + {B8554582-DE19-41A2-9784-9B27C9F22429}.Release|x86.Build.0 = Release|Any CPU {A37994CC-A23A-415E-8B61-9468C7178A55}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {A37994CC-A23A-415E-8B61-9468C7178A55}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A37994CC-A23A-415E-8B61-9468C7178A55}.Debug|x64.ActiveCfg = Debug|Any CPU + {A37994CC-A23A-415E-8B61-9468C7178A55}.Debug|x64.Build.0 = Debug|Any CPU + {A37994CC-A23A-415E-8B61-9468C7178A55}.Debug|x86.ActiveCfg = Debug|Any CPU + {A37994CC-A23A-415E-8B61-9468C7178A55}.Debug|x86.Build.0 = Debug|Any CPU {A37994CC-A23A-415E-8B61-9468C7178A55}.Release|Any CPU.ActiveCfg = Release|Any CPU {A37994CC-A23A-415E-8B61-9468C7178A55}.Release|Any CPU.Build.0 = Release|Any CPU + {A37994CC-A23A-415E-8B61-9468C7178A55}.Release|x64.ActiveCfg = Release|Any CPU + {A37994CC-A23A-415E-8B61-9468C7178A55}.Release|x64.Build.0 = Release|Any CPU + {A37994CC-A23A-415E-8B61-9468C7178A55}.Release|x86.ActiveCfg = Release|Any CPU + {A37994CC-A23A-415E-8B61-9468C7178A55}.Release|x86.Build.0 = Release|Any CPU {6DAAAA87-8DDF-4E60-81CE-D8900327DE33}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {6DAAAA87-8DDF-4E60-81CE-D8900327DE33}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6DAAAA87-8DDF-4E60-81CE-D8900327DE33}.Debug|x64.ActiveCfg = Debug|Any CPU + {6DAAAA87-8DDF-4E60-81CE-D8900327DE33}.Debug|x64.Build.0 = Debug|Any CPU + {6DAAAA87-8DDF-4E60-81CE-D8900327DE33}.Debug|x86.ActiveCfg = Debug|Any CPU + {6DAAAA87-8DDF-4E60-81CE-D8900327DE33}.Debug|x86.Build.0 = Debug|Any CPU {6DAAAA87-8DDF-4E60-81CE-D8900327DE33}.Release|Any CPU.ActiveCfg = Release|Any CPU {6DAAAA87-8DDF-4E60-81CE-D8900327DE33}.Release|Any CPU.Build.0 = Release|Any CPU + {6DAAAA87-8DDF-4E60-81CE-D8900327DE33}.Release|x64.ActiveCfg = Release|Any CPU + {6DAAAA87-8DDF-4E60-81CE-D8900327DE33}.Release|x64.Build.0 = Release|Any CPU + {6DAAAA87-8DDF-4E60-81CE-D8900327DE33}.Release|x86.ActiveCfg = Release|Any CPU + {6DAAAA87-8DDF-4E60-81CE-D8900327DE33}.Release|x86.Build.0 = Release|Any CPU {227C88B1-0510-4010-B142-C44725578FCD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {227C88B1-0510-4010-B142-C44725578FCD}.Debug|Any CPU.Build.0 = Debug|Any CPU + {227C88B1-0510-4010-B142-C44725578FCD}.Debug|x64.ActiveCfg = Debug|Any CPU + {227C88B1-0510-4010-B142-C44725578FCD}.Debug|x64.Build.0 = Debug|Any CPU + {227C88B1-0510-4010-B142-C44725578FCD}.Debug|x86.ActiveCfg = Debug|Any CPU + {227C88B1-0510-4010-B142-C44725578FCD}.Debug|x86.Build.0 = Debug|Any CPU {227C88B1-0510-4010-B142-C44725578FCD}.Release|Any CPU.ActiveCfg = Release|Any CPU {227C88B1-0510-4010-B142-C44725578FCD}.Release|Any CPU.Build.0 = Release|Any CPU + {227C88B1-0510-4010-B142-C44725578FCD}.Release|x64.ActiveCfg = Release|Any CPU + {227C88B1-0510-4010-B142-C44725578FCD}.Release|x64.Build.0 = Release|Any CPU + {227C88B1-0510-4010-B142-C44725578FCD}.Release|x86.ActiveCfg = Release|Any CPU + {227C88B1-0510-4010-B142-C44725578FCD}.Release|x86.Build.0 = Release|Any CPU {8A676AAA-FEE3-4C18-870A-66E59AD9069F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {8A676AAA-FEE3-4C18-870A-66E59AD9069F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8A676AAA-FEE3-4C18-870A-66E59AD9069F}.Debug|x64.ActiveCfg = Debug|Any CPU + {8A676AAA-FEE3-4C18-870A-66E59AD9069F}.Debug|x64.Build.0 = Debug|Any CPU + {8A676AAA-FEE3-4C18-870A-66E59AD9069F}.Debug|x86.ActiveCfg = Debug|Any CPU + {8A676AAA-FEE3-4C18-870A-66E59AD9069F}.Debug|x86.Build.0 = Debug|Any CPU {8A676AAA-FEE3-4C18-870A-66E59AD9069F}.Release|Any CPU.ActiveCfg = Release|Any CPU {8A676AAA-FEE3-4C18-870A-66E59AD9069F}.Release|Any CPU.Build.0 = Release|Any CPU + {8A676AAA-FEE3-4C18-870A-66E59AD9069F}.Release|x64.ActiveCfg = Release|Any CPU + {8A676AAA-FEE3-4C18-870A-66E59AD9069F}.Release|x64.Build.0 = Release|Any CPU + {8A676AAA-FEE3-4C18-870A-66E59AD9069F}.Release|x86.ActiveCfg = Release|Any CPU + {8A676AAA-FEE3-4C18-870A-66E59AD9069F}.Release|x86.Build.0 = Release|Any CPU {9521D9E0-642A-4C7E-BD10-372DF235CF62}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {9521D9E0-642A-4C7E-BD10-372DF235CF62}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9521D9E0-642A-4C7E-BD10-372DF235CF62}.Debug|x64.ActiveCfg = Debug|Any CPU + {9521D9E0-642A-4C7E-BD10-372DF235CF62}.Debug|x64.Build.0 = Debug|Any CPU + {9521D9E0-642A-4C7E-BD10-372DF235CF62}.Debug|x86.ActiveCfg = Debug|Any CPU + {9521D9E0-642A-4C7E-BD10-372DF235CF62}.Debug|x86.Build.0 = Debug|Any CPU {9521D9E0-642A-4C7E-BD10-372DF235CF62}.Release|Any CPU.ActiveCfg = Release|Any CPU {9521D9E0-642A-4C7E-BD10-372DF235CF62}.Release|Any CPU.Build.0 = Release|Any CPU + {9521D9E0-642A-4C7E-BD10-372DF235CF62}.Release|x64.ActiveCfg = Release|Any CPU + {9521D9E0-642A-4C7E-BD10-372DF235CF62}.Release|x64.Build.0 = Release|Any CPU + {9521D9E0-642A-4C7E-BD10-372DF235CF62}.Release|x86.ActiveCfg = Release|Any CPU + {9521D9E0-642A-4C7E-BD10-372DF235CF62}.Release|x86.Build.0 = Release|Any CPU {8C1E83E5-852F-447F-95B3-0CD2199F3A66}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {8C1E83E5-852F-447F-95B3-0CD2199F3A66}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8C1E83E5-852F-447F-95B3-0CD2199F3A66}.Debug|x64.ActiveCfg = Debug|Any CPU + {8C1E83E5-852F-447F-95B3-0CD2199F3A66}.Debug|x64.Build.0 = Debug|Any CPU + {8C1E83E5-852F-447F-95B3-0CD2199F3A66}.Debug|x86.ActiveCfg = Debug|Any CPU + {8C1E83E5-852F-447F-95B3-0CD2199F3A66}.Debug|x86.Build.0 = Debug|Any CPU {8C1E83E5-852F-447F-95B3-0CD2199F3A66}.Release|Any CPU.ActiveCfg = Release|Any CPU {8C1E83E5-852F-447F-95B3-0CD2199F3A66}.Release|Any CPU.Build.0 = Release|Any CPU + {8C1E83E5-852F-447F-95B3-0CD2199F3A66}.Release|x64.ActiveCfg = Release|Any CPU + {8C1E83E5-852F-447F-95B3-0CD2199F3A66}.Release|x64.Build.0 = Release|Any CPU + {8C1E83E5-852F-447F-95B3-0CD2199F3A66}.Release|x86.ActiveCfg = Release|Any CPU + {8C1E83E5-852F-447F-95B3-0CD2199F3A66}.Release|x86.Build.0 = Release|Any CPU {D8E29F7F-8F7C-462D-8E5F-7C0DAA08D527}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {D8E29F7F-8F7C-462D-8E5F-7C0DAA08D527}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D8E29F7F-8F7C-462D-8E5F-7C0DAA08D527}.Debug|x64.ActiveCfg = Debug|Any CPU + {D8E29F7F-8F7C-462D-8E5F-7C0DAA08D527}.Debug|x64.Build.0 = Debug|Any CPU + {D8E29F7F-8F7C-462D-8E5F-7C0DAA08D527}.Debug|x86.ActiveCfg = Debug|Any CPU + {D8E29F7F-8F7C-462D-8E5F-7C0DAA08D527}.Debug|x86.Build.0 = Debug|Any CPU {D8E29F7F-8F7C-462D-8E5F-7C0DAA08D527}.Release|Any CPU.ActiveCfg = Release|Any CPU {D8E29F7F-8F7C-462D-8E5F-7C0DAA08D527}.Release|Any CPU.Build.0 = Release|Any CPU + {D8E29F7F-8F7C-462D-8E5F-7C0DAA08D527}.Release|x64.ActiveCfg = Release|Any CPU + {D8E29F7F-8F7C-462D-8E5F-7C0DAA08D527}.Release|x64.Build.0 = Release|Any CPU + {D8E29F7F-8F7C-462D-8E5F-7C0DAA08D527}.Release|x86.ActiveCfg = Release|Any CPU + {D8E29F7F-8F7C-462D-8E5F-7C0DAA08D527}.Release|x86.Build.0 = Release|Any CPU {6C039E97-4AFA-4FB7-B361-C40001A632B2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {6C039E97-4AFA-4FB7-B361-C40001A632B2}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6C039E97-4AFA-4FB7-B361-C40001A632B2}.Debug|x64.ActiveCfg = Debug|Any CPU + {6C039E97-4AFA-4FB7-B361-C40001A632B2}.Debug|x64.Build.0 = Debug|Any CPU + {6C039E97-4AFA-4FB7-B361-C40001A632B2}.Debug|x86.ActiveCfg = Debug|Any CPU + {6C039E97-4AFA-4FB7-B361-C40001A632B2}.Debug|x86.Build.0 = Debug|Any CPU {6C039E97-4AFA-4FB7-B361-C40001A632B2}.Release|Any CPU.ActiveCfg = Release|Any CPU {6C039E97-4AFA-4FB7-B361-C40001A632B2}.Release|Any CPU.Build.0 = Release|Any CPU + {6C039E97-4AFA-4FB7-B361-C40001A632B2}.Release|x64.ActiveCfg = Release|Any CPU + {6C039E97-4AFA-4FB7-B361-C40001A632B2}.Release|x64.Build.0 = Release|Any CPU + {6C039E97-4AFA-4FB7-B361-C40001A632B2}.Release|x86.ActiveCfg = Release|Any CPU + {6C039E97-4AFA-4FB7-B361-C40001A632B2}.Release|x86.Build.0 = Release|Any CPU {1A8222A7-699E-493E-852B-87646FD7C311}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {1A8222A7-699E-493E-852B-87646FD7C311}.Debug|Any CPU.Build.0 = Debug|Any CPU + {1A8222A7-699E-493E-852B-87646FD7C311}.Debug|x64.ActiveCfg = Debug|Any CPU + {1A8222A7-699E-493E-852B-87646FD7C311}.Debug|x64.Build.0 = Debug|Any CPU + {1A8222A7-699E-493E-852B-87646FD7C311}.Debug|x86.ActiveCfg = Debug|Any CPU + {1A8222A7-699E-493E-852B-87646FD7C311}.Debug|x86.Build.0 = Debug|Any CPU {1A8222A7-699E-493E-852B-87646FD7C311}.Release|Any CPU.ActiveCfg = Release|Any CPU {1A8222A7-699E-493E-852B-87646FD7C311}.Release|Any CPU.Build.0 = Release|Any CPU + {1A8222A7-699E-493E-852B-87646FD7C311}.Release|x64.ActiveCfg = Release|Any CPU + {1A8222A7-699E-493E-852B-87646FD7C311}.Release|x64.Build.0 = Release|Any CPU + {1A8222A7-699E-493E-852B-87646FD7C311}.Release|x86.ActiveCfg = Release|Any CPU + {1A8222A7-699E-493E-852B-87646FD7C311}.Release|x86.Build.0 = Release|Any CPU {A7B6B97C-278B-4593-9967-6D791045C0B6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {A7B6B97C-278B-4593-9967-6D791045C0B6}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A7B6B97C-278B-4593-9967-6D791045C0B6}.Debug|x64.ActiveCfg = Debug|Any CPU + {A7B6B97C-278B-4593-9967-6D791045C0B6}.Debug|x64.Build.0 = Debug|Any CPU + {A7B6B97C-278B-4593-9967-6D791045C0B6}.Debug|x86.ActiveCfg = Debug|Any CPU + {A7B6B97C-278B-4593-9967-6D791045C0B6}.Debug|x86.Build.0 = Debug|Any CPU {A7B6B97C-278B-4593-9967-6D791045C0B6}.Release|Any CPU.ActiveCfg = Release|Any CPU {A7B6B97C-278B-4593-9967-6D791045C0B6}.Release|Any CPU.Build.0 = Release|Any CPU + {A7B6B97C-278B-4593-9967-6D791045C0B6}.Release|x64.ActiveCfg = Release|Any CPU + {A7B6B97C-278B-4593-9967-6D791045C0B6}.Release|x64.Build.0 = Release|Any CPU + {A7B6B97C-278B-4593-9967-6D791045C0B6}.Release|x86.ActiveCfg = Release|Any CPU + {A7B6B97C-278B-4593-9967-6D791045C0B6}.Release|x86.Build.0 = Release|Any CPU {94AF0E32-E0F7-417D-A010-0545964641F4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {94AF0E32-E0F7-417D-A010-0545964641F4}.Debug|Any CPU.Build.0 = Debug|Any CPU + {94AF0E32-E0F7-417D-A010-0545964641F4}.Debug|x64.ActiveCfg = Debug|Any CPU + {94AF0E32-E0F7-417D-A010-0545964641F4}.Debug|x64.Build.0 = Debug|Any CPU + {94AF0E32-E0F7-417D-A010-0545964641F4}.Debug|x86.ActiveCfg = Debug|Any CPU + {94AF0E32-E0F7-417D-A010-0545964641F4}.Debug|x86.Build.0 = Debug|Any CPU {94AF0E32-E0F7-417D-A010-0545964641F4}.Release|Any CPU.ActiveCfg = Release|Any CPU {94AF0E32-E0F7-417D-A010-0545964641F4}.Release|Any CPU.Build.0 = Release|Any CPU + {94AF0E32-E0F7-417D-A010-0545964641F4}.Release|x64.ActiveCfg = Release|Any CPU + {94AF0E32-E0F7-417D-A010-0545964641F4}.Release|x64.Build.0 = Release|Any CPU + {94AF0E32-E0F7-417D-A010-0545964641F4}.Release|x86.ActiveCfg = Release|Any CPU + {94AF0E32-E0F7-417D-A010-0545964641F4}.Release|x86.Build.0 = Release|Any CPU {C075FACD-83A3-48F2-9F6E-D181F66FB2EC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {C075FACD-83A3-48F2-9F6E-D181F66FB2EC}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C075FACD-83A3-48F2-9F6E-D181F66FB2EC}.Debug|x64.ActiveCfg = Debug|Any CPU + {C075FACD-83A3-48F2-9F6E-D181F66FB2EC}.Debug|x64.Build.0 = Debug|Any CPU + {C075FACD-83A3-48F2-9F6E-D181F66FB2EC}.Debug|x86.ActiveCfg = Debug|Any CPU + {C075FACD-83A3-48F2-9F6E-D181F66FB2EC}.Debug|x86.Build.0 = Debug|Any CPU {C075FACD-83A3-48F2-9F6E-D181F66FB2EC}.Release|Any CPU.ActiveCfg = Release|Any CPU {C075FACD-83A3-48F2-9F6E-D181F66FB2EC}.Release|Any CPU.Build.0 = Release|Any CPU + {C075FACD-83A3-48F2-9F6E-D181F66FB2EC}.Release|x64.ActiveCfg = Release|Any CPU + {C075FACD-83A3-48F2-9F6E-D181F66FB2EC}.Release|x64.Build.0 = Release|Any CPU + {C075FACD-83A3-48F2-9F6E-D181F66FB2EC}.Release|x86.ActiveCfg = Release|Any CPU + {C075FACD-83A3-48F2-9F6E-D181F66FB2EC}.Release|x86.Build.0 = Release|Any CPU {4501D331-9428-48AC-8C89-52034BF7F06F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {4501D331-9428-48AC-8C89-52034BF7F06F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4501D331-9428-48AC-8C89-52034BF7F06F}.Debug|x64.ActiveCfg = Debug|Any CPU + {4501D331-9428-48AC-8C89-52034BF7F06F}.Debug|x64.Build.0 = Debug|Any CPU + {4501D331-9428-48AC-8C89-52034BF7F06F}.Debug|x86.ActiveCfg = Debug|Any CPU + {4501D331-9428-48AC-8C89-52034BF7F06F}.Debug|x86.Build.0 = Debug|Any CPU {4501D331-9428-48AC-8C89-52034BF7F06F}.Release|Any CPU.ActiveCfg = Release|Any CPU {4501D331-9428-48AC-8C89-52034BF7F06F}.Release|Any CPU.Build.0 = Release|Any CPU + {4501D331-9428-48AC-8C89-52034BF7F06F}.Release|x64.ActiveCfg = Release|Any CPU + {4501D331-9428-48AC-8C89-52034BF7F06F}.Release|x64.Build.0 = Release|Any CPU + {4501D331-9428-48AC-8C89-52034BF7F06F}.Release|x86.ActiveCfg = Release|Any CPU + {4501D331-9428-48AC-8C89-52034BF7F06F}.Release|x86.Build.0 = Release|Any CPU {92AB528E-2BE4-438F-9C0E-83BB3D967403}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {92AB528E-2BE4-438F-9C0E-83BB3D967403}.Debug|Any CPU.Build.0 = Debug|Any CPU + {92AB528E-2BE4-438F-9C0E-83BB3D967403}.Debug|x64.ActiveCfg = Debug|Any CPU + {92AB528E-2BE4-438F-9C0E-83BB3D967403}.Debug|x64.Build.0 = Debug|Any CPU + {92AB528E-2BE4-438F-9C0E-83BB3D967403}.Debug|x86.ActiveCfg = Debug|Any CPU + {92AB528E-2BE4-438F-9C0E-83BB3D967403}.Debug|x86.Build.0 = Debug|Any CPU {92AB528E-2BE4-438F-9C0E-83BB3D967403}.Release|Any CPU.ActiveCfg = Release|Any CPU {92AB528E-2BE4-438F-9C0E-83BB3D967403}.Release|Any CPU.Build.0 = Release|Any CPU + {92AB528E-2BE4-438F-9C0E-83BB3D967403}.Release|x64.ActiveCfg = Release|Any CPU + {92AB528E-2BE4-438F-9C0E-83BB3D967403}.Release|x64.Build.0 = Release|Any CPU + {92AB528E-2BE4-438F-9C0E-83BB3D967403}.Release|x86.ActiveCfg = Release|Any CPU + {92AB528E-2BE4-438F-9C0E-83BB3D967403}.Release|x86.Build.0 = Release|Any CPU {8E7B180C-7057-4C81-BD0F-D35A74501FCB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {8E7B180C-7057-4C81-BD0F-D35A74501FCB}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8E7B180C-7057-4C81-BD0F-D35A74501FCB}.Debug|x64.ActiveCfg = Debug|Any CPU + {8E7B180C-7057-4C81-BD0F-D35A74501FCB}.Debug|x64.Build.0 = Debug|Any CPU + {8E7B180C-7057-4C81-BD0F-D35A74501FCB}.Debug|x86.ActiveCfg = Debug|Any CPU + {8E7B180C-7057-4C81-BD0F-D35A74501FCB}.Debug|x86.Build.0 = Debug|Any CPU {8E7B180C-7057-4C81-BD0F-D35A74501FCB}.Release|Any CPU.ActiveCfg = Release|Any CPU {8E7B180C-7057-4C81-BD0F-D35A74501FCB}.Release|Any CPU.Build.0 = Release|Any CPU + {8E7B180C-7057-4C81-BD0F-D35A74501FCB}.Release|x64.ActiveCfg = Release|Any CPU + {8E7B180C-7057-4C81-BD0F-D35A74501FCB}.Release|x64.Build.0 = Release|Any CPU + {8E7B180C-7057-4C81-BD0F-D35A74501FCB}.Release|x86.ActiveCfg = Release|Any CPU + {8E7B180C-7057-4C81-BD0F-D35A74501FCB}.Release|x86.Build.0 = Release|Any CPU {89162E83-DEFA-4474-B6AE-669EB64374DB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {89162E83-DEFA-4474-B6AE-669EB64374DB}.Debug|Any CPU.Build.0 = Debug|Any CPU + {89162E83-DEFA-4474-B6AE-669EB64374DB}.Debug|x64.ActiveCfg = Debug|Any CPU + {89162E83-DEFA-4474-B6AE-669EB64374DB}.Debug|x64.Build.0 = Debug|Any CPU + {89162E83-DEFA-4474-B6AE-669EB64374DB}.Debug|x86.ActiveCfg = Debug|Any CPU + {89162E83-DEFA-4474-B6AE-669EB64374DB}.Debug|x86.Build.0 = Debug|Any CPU {89162E83-DEFA-4474-B6AE-669EB64374DB}.Release|Any CPU.ActiveCfg = Release|Any CPU {89162E83-DEFA-4474-B6AE-669EB64374DB}.Release|Any CPU.Build.0 = Release|Any CPU + {89162E83-DEFA-4474-B6AE-669EB64374DB}.Release|x64.ActiveCfg = Release|Any CPU + {89162E83-DEFA-4474-B6AE-669EB64374DB}.Release|x64.Build.0 = Release|Any CPU + {89162E83-DEFA-4474-B6AE-669EB64374DB}.Release|x86.ActiveCfg = Release|Any CPU + {89162E83-DEFA-4474-B6AE-669EB64374DB}.Release|x86.Build.0 = Release|Any CPU {C498BDCC-CD27-4867-85AA-00EFA5EE3809}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {C498BDCC-CD27-4867-85AA-00EFA5EE3809}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C498BDCC-CD27-4867-85AA-00EFA5EE3809}.Debug|x64.ActiveCfg = Debug|Any CPU + {C498BDCC-CD27-4867-85AA-00EFA5EE3809}.Debug|x64.Build.0 = Debug|Any CPU + {C498BDCC-CD27-4867-85AA-00EFA5EE3809}.Debug|x86.ActiveCfg = Debug|Any CPU + {C498BDCC-CD27-4867-85AA-00EFA5EE3809}.Debug|x86.Build.0 = Debug|Any CPU {C498BDCC-CD27-4867-85AA-00EFA5EE3809}.Release|Any CPU.ActiveCfg = Release|Any CPU {C498BDCC-CD27-4867-85AA-00EFA5EE3809}.Release|Any CPU.Build.0 = Release|Any CPU + {C498BDCC-CD27-4867-85AA-00EFA5EE3809}.Release|x64.ActiveCfg = Release|Any CPU + {C498BDCC-CD27-4867-85AA-00EFA5EE3809}.Release|x64.Build.0 = Release|Any CPU + {C498BDCC-CD27-4867-85AA-00EFA5EE3809}.Release|x86.ActiveCfg = Release|Any CPU + {C498BDCC-CD27-4867-85AA-00EFA5EE3809}.Release|x86.Build.0 = Release|Any CPU {B86AF861-EB06-45A7-9132-E4DDB498C98C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {B86AF861-EB06-45A7-9132-E4DDB498C98C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B86AF861-EB06-45A7-9132-E4DDB498C98C}.Debug|x64.ActiveCfg = Debug|Any CPU + {B86AF861-EB06-45A7-9132-E4DDB498C98C}.Debug|x64.Build.0 = Debug|Any CPU + {B86AF861-EB06-45A7-9132-E4DDB498C98C}.Debug|x86.ActiveCfg = Debug|Any CPU + {B86AF861-EB06-45A7-9132-E4DDB498C98C}.Debug|x86.Build.0 = Debug|Any CPU {B86AF861-EB06-45A7-9132-E4DDB498C98C}.Release|Any CPU.ActiveCfg = Release|Any CPU {B86AF861-EB06-45A7-9132-E4DDB498C98C}.Release|Any CPU.Build.0 = Release|Any CPU + {B86AF861-EB06-45A7-9132-E4DDB498C98C}.Release|x64.ActiveCfg = Release|Any CPU + {B86AF861-EB06-45A7-9132-E4DDB498C98C}.Release|x64.Build.0 = Release|Any CPU + {B86AF861-EB06-45A7-9132-E4DDB498C98C}.Release|x86.ActiveCfg = Release|Any CPU + {B86AF861-EB06-45A7-9132-E4DDB498C98C}.Release|x86.Build.0 = Release|Any CPU {1B8B170B-4FD8-4CAE-97E3-4351DBD7939A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {1B8B170B-4FD8-4CAE-97E3-4351DBD7939A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {1B8B170B-4FD8-4CAE-97E3-4351DBD7939A}.Debug|x64.ActiveCfg = Debug|Any CPU + {1B8B170B-4FD8-4CAE-97E3-4351DBD7939A}.Debug|x64.Build.0 = Debug|Any CPU + {1B8B170B-4FD8-4CAE-97E3-4351DBD7939A}.Debug|x86.ActiveCfg = Debug|Any CPU + {1B8B170B-4FD8-4CAE-97E3-4351DBD7939A}.Debug|x86.Build.0 = Debug|Any CPU {1B8B170B-4FD8-4CAE-97E3-4351DBD7939A}.Release|Any CPU.ActiveCfg = Release|Any CPU {1B8B170B-4FD8-4CAE-97E3-4351DBD7939A}.Release|Any CPU.Build.0 = Release|Any CPU + {1B8B170B-4FD8-4CAE-97E3-4351DBD7939A}.Release|x64.ActiveCfg = Release|Any CPU + {1B8B170B-4FD8-4CAE-97E3-4351DBD7939A}.Release|x64.Build.0 = Release|Any CPU + {1B8B170B-4FD8-4CAE-97E3-4351DBD7939A}.Release|x86.ActiveCfg = Release|Any CPU + {1B8B170B-4FD8-4CAE-97E3-4351DBD7939A}.Release|x86.Build.0 = Release|Any CPU {8ED5D3DC-86F7-4599-8ED5-C0E20FFDE7F6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {8ED5D3DC-86F7-4599-8ED5-C0E20FFDE7F6}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8ED5D3DC-86F7-4599-8ED5-C0E20FFDE7F6}.Debug|x64.ActiveCfg = Debug|Any CPU + {8ED5D3DC-86F7-4599-8ED5-C0E20FFDE7F6}.Debug|x64.Build.0 = Debug|Any CPU + {8ED5D3DC-86F7-4599-8ED5-C0E20FFDE7F6}.Debug|x86.ActiveCfg = Debug|Any CPU + {8ED5D3DC-86F7-4599-8ED5-C0E20FFDE7F6}.Debug|x86.Build.0 = Debug|Any CPU {8ED5D3DC-86F7-4599-8ED5-C0E20FFDE7F6}.Release|Any CPU.ActiveCfg = Release|Any CPU {8ED5D3DC-86F7-4599-8ED5-C0E20FFDE7F6}.Release|Any CPU.Build.0 = Release|Any CPU + {8ED5D3DC-86F7-4599-8ED5-C0E20FFDE7F6}.Release|x64.ActiveCfg = Release|Any CPU + {8ED5D3DC-86F7-4599-8ED5-C0E20FFDE7F6}.Release|x64.Build.0 = Release|Any CPU + {8ED5D3DC-86F7-4599-8ED5-C0E20FFDE7F6}.Release|x86.ActiveCfg = Release|Any CPU + {8ED5D3DC-86F7-4599-8ED5-C0E20FFDE7F6}.Release|x86.Build.0 = Release|Any CPU {0527FB48-60E0-4D38-B3D9-0A889E1FFAB2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {0527FB48-60E0-4D38-B3D9-0A889E1FFAB2}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0527FB48-60E0-4D38-B3D9-0A889E1FFAB2}.Debug|x64.ActiveCfg = Debug|Any CPU + {0527FB48-60E0-4D38-B3D9-0A889E1FFAB2}.Debug|x64.Build.0 = Debug|Any CPU + {0527FB48-60E0-4D38-B3D9-0A889E1FFAB2}.Debug|x86.ActiveCfg = Debug|Any CPU + {0527FB48-60E0-4D38-B3D9-0A889E1FFAB2}.Debug|x86.Build.0 = Debug|Any CPU {0527FB48-60E0-4D38-B3D9-0A889E1FFAB2}.Release|Any CPU.ActiveCfg = Release|Any CPU {0527FB48-60E0-4D38-B3D9-0A889E1FFAB2}.Release|Any CPU.Build.0 = Release|Any CPU + {0527FB48-60E0-4D38-B3D9-0A889E1FFAB2}.Release|x64.ActiveCfg = Release|Any CPU + {0527FB48-60E0-4D38-B3D9-0A889E1FFAB2}.Release|x64.Build.0 = Release|Any CPU + {0527FB48-60E0-4D38-B3D9-0A889E1FFAB2}.Release|x86.ActiveCfg = Release|Any CPU + {0527FB48-60E0-4D38-B3D9-0A889E1FFAB2}.Release|x86.Build.0 = Release|Any CPU {1EA44258-04FC-48D5-A60D-AE92214DEB1A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {1EA44258-04FC-48D5-A60D-AE92214DEB1A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {1EA44258-04FC-48D5-A60D-AE92214DEB1A}.Debug|x64.ActiveCfg = Debug|Any CPU + {1EA44258-04FC-48D5-A60D-AE92214DEB1A}.Debug|x64.Build.0 = Debug|Any CPU + {1EA44258-04FC-48D5-A60D-AE92214DEB1A}.Debug|x86.ActiveCfg = Debug|Any CPU + {1EA44258-04FC-48D5-A60D-AE92214DEB1A}.Debug|x86.Build.0 = Debug|Any CPU {1EA44258-04FC-48D5-A60D-AE92214DEB1A}.Release|Any CPU.ActiveCfg = Release|Any CPU {1EA44258-04FC-48D5-A60D-AE92214DEB1A}.Release|Any CPU.Build.0 = Release|Any CPU + {1EA44258-04FC-48D5-A60D-AE92214DEB1A}.Release|x64.ActiveCfg = Release|Any CPU + {1EA44258-04FC-48D5-A60D-AE92214DEB1A}.Release|x64.Build.0 = Release|Any CPU + {1EA44258-04FC-48D5-A60D-AE92214DEB1A}.Release|x86.ActiveCfg = Release|Any CPU + {1EA44258-04FC-48D5-A60D-AE92214DEB1A}.Release|x86.Build.0 = Release|Any CPU {0B6CBF18-D11C-4CC3-A50C-437A9F7E761D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {0B6CBF18-D11C-4CC3-A50C-437A9F7E761D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0B6CBF18-D11C-4CC3-A50C-437A9F7E761D}.Debug|x64.ActiveCfg = Debug|Any CPU + {0B6CBF18-D11C-4CC3-A50C-437A9F7E761D}.Debug|x64.Build.0 = Debug|Any CPU + {0B6CBF18-D11C-4CC3-A50C-437A9F7E761D}.Debug|x86.ActiveCfg = Debug|Any CPU + {0B6CBF18-D11C-4CC3-A50C-437A9F7E761D}.Debug|x86.Build.0 = Debug|Any CPU {0B6CBF18-D11C-4CC3-A50C-437A9F7E761D}.Release|Any CPU.ActiveCfg = Release|Any CPU {0B6CBF18-D11C-4CC3-A50C-437A9F7E761D}.Release|Any CPU.Build.0 = Release|Any CPU + {0B6CBF18-D11C-4CC3-A50C-437A9F7E761D}.Release|x64.ActiveCfg = Release|Any CPU + {0B6CBF18-D11C-4CC3-A50C-437A9F7E761D}.Release|x64.Build.0 = Release|Any CPU + {0B6CBF18-D11C-4CC3-A50C-437A9F7E761D}.Release|x86.ActiveCfg = Release|Any CPU + {0B6CBF18-D11C-4CC3-A50C-437A9F7E761D}.Release|x86.Build.0 = Release|Any CPU {1F6C87C7-1911-4E96-ADB4-1EF27BD09C60}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {1F6C87C7-1911-4E96-ADB4-1EF27BD09C60}.Debug|Any CPU.Build.0 = Debug|Any CPU + {1F6C87C7-1911-4E96-ADB4-1EF27BD09C60}.Debug|x64.ActiveCfg = Debug|Any CPU + {1F6C87C7-1911-4E96-ADB4-1EF27BD09C60}.Debug|x64.Build.0 = Debug|Any CPU + {1F6C87C7-1911-4E96-ADB4-1EF27BD09C60}.Debug|x86.ActiveCfg = Debug|Any CPU + {1F6C87C7-1911-4E96-ADB4-1EF27BD09C60}.Debug|x86.Build.0 = Debug|Any CPU {1F6C87C7-1911-4E96-ADB4-1EF27BD09C60}.Release|Any CPU.ActiveCfg = Release|Any CPU {1F6C87C7-1911-4E96-ADB4-1EF27BD09C60}.Release|Any CPU.Build.0 = Release|Any CPU + {1F6C87C7-1911-4E96-ADB4-1EF27BD09C60}.Release|x64.ActiveCfg = Release|Any CPU + {1F6C87C7-1911-4E96-ADB4-1EF27BD09C60}.Release|x64.Build.0 = Release|Any CPU + {1F6C87C7-1911-4E96-ADB4-1EF27BD09C60}.Release|x86.ActiveCfg = Release|Any CPU + {1F6C87C7-1911-4E96-ADB4-1EF27BD09C60}.Release|x86.Build.0 = Release|Any CPU {71C2BC96-C104-4748-B6CF-E74BAC4E709D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {71C2BC96-C104-4748-B6CF-E74BAC4E709D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {71C2BC96-C104-4748-B6CF-E74BAC4E709D}.Debug|x64.ActiveCfg = Debug|Any CPU + {71C2BC96-C104-4748-B6CF-E74BAC4E709D}.Debug|x64.Build.0 = Debug|Any CPU + {71C2BC96-C104-4748-B6CF-E74BAC4E709D}.Debug|x86.ActiveCfg = Debug|Any CPU + {71C2BC96-C104-4748-B6CF-E74BAC4E709D}.Debug|x86.Build.0 = Debug|Any CPU {71C2BC96-C104-4748-B6CF-E74BAC4E709D}.Release|Any CPU.ActiveCfg = Release|Any CPU {71C2BC96-C104-4748-B6CF-E74BAC4E709D}.Release|Any CPU.Build.0 = Release|Any CPU + {71C2BC96-C104-4748-B6CF-E74BAC4E709D}.Release|x64.ActiveCfg = Release|Any CPU + {71C2BC96-C104-4748-B6CF-E74BAC4E709D}.Release|x64.Build.0 = Release|Any CPU + {71C2BC96-C104-4748-B6CF-E74BAC4E709D}.Release|x86.ActiveCfg = Release|Any CPU + {71C2BC96-C104-4748-B6CF-E74BAC4E709D}.Release|x86.Build.0 = Release|Any CPU {21649885-B40A-4F6B-9286-A7E378276C3E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {21649885-B40A-4F6B-9286-A7E378276C3E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {21649885-B40A-4F6B-9286-A7E378276C3E}.Debug|x64.ActiveCfg = Debug|Any CPU + {21649885-B40A-4F6B-9286-A7E378276C3E}.Debug|x64.Build.0 = Debug|Any CPU + {21649885-B40A-4F6B-9286-A7E378276C3E}.Debug|x86.ActiveCfg = Debug|Any CPU + {21649885-B40A-4F6B-9286-A7E378276C3E}.Debug|x86.Build.0 = Debug|Any CPU {21649885-B40A-4F6B-9286-A7E378276C3E}.Release|Any CPU.ActiveCfg = Release|Any CPU {21649885-B40A-4F6B-9286-A7E378276C3E}.Release|Any CPU.Build.0 = Release|Any CPU + {21649885-B40A-4F6B-9286-A7E378276C3E}.Release|x64.ActiveCfg = Release|Any CPU + {21649885-B40A-4F6B-9286-A7E378276C3E}.Release|x64.Build.0 = Release|Any CPU + {21649885-B40A-4F6B-9286-A7E378276C3E}.Release|x86.ActiveCfg = Release|Any CPU + {21649885-B40A-4F6B-9286-A7E378276C3E}.Release|x86.Build.0 = Release|Any CPU {DA8E9EC2-990C-4DA2-A3F3-7E7796C71DD8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {DA8E9EC2-990C-4DA2-A3F3-7E7796C71DD8}.Debug|Any CPU.Build.0 = Debug|Any CPU + {DA8E9EC2-990C-4DA2-A3F3-7E7796C71DD8}.Debug|x64.ActiveCfg = Debug|Any CPU + {DA8E9EC2-990C-4DA2-A3F3-7E7796C71DD8}.Debug|x64.Build.0 = Debug|Any CPU + {DA8E9EC2-990C-4DA2-A3F3-7E7796C71DD8}.Debug|x86.ActiveCfg = Debug|Any CPU + {DA8E9EC2-990C-4DA2-A3F3-7E7796C71DD8}.Debug|x86.Build.0 = Debug|Any CPU {DA8E9EC2-990C-4DA2-A3F3-7E7796C71DD8}.Release|Any CPU.ActiveCfg = Release|Any CPU {DA8E9EC2-990C-4DA2-A3F3-7E7796C71DD8}.Release|Any CPU.Build.0 = Release|Any CPU + {DA8E9EC2-990C-4DA2-A3F3-7E7796C71DD8}.Release|x64.ActiveCfg = Release|Any CPU + {DA8E9EC2-990C-4DA2-A3F3-7E7796C71DD8}.Release|x64.Build.0 = Release|Any CPU + {DA8E9EC2-990C-4DA2-A3F3-7E7796C71DD8}.Release|x86.ActiveCfg = Release|Any CPU + {DA8E9EC2-990C-4DA2-A3F3-7E7796C71DD8}.Release|x86.Build.0 = Release|Any CPU + {B54702BF-ECAD-4F96-BC43-F5B670F0A668}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B54702BF-ECAD-4F96-BC43-F5B670F0A668}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B54702BF-ECAD-4F96-BC43-F5B670F0A668}.Debug|x64.ActiveCfg = Debug|Any CPU + {B54702BF-ECAD-4F96-BC43-F5B670F0A668}.Debug|x64.Build.0 = Debug|Any CPU + {B54702BF-ECAD-4F96-BC43-F5B670F0A668}.Debug|x86.ActiveCfg = Debug|Any CPU + {B54702BF-ECAD-4F96-BC43-F5B670F0A668}.Debug|x86.Build.0 = Debug|Any CPU + {B54702BF-ECAD-4F96-BC43-F5B670F0A668}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B54702BF-ECAD-4F96-BC43-F5B670F0A668}.Release|Any CPU.Build.0 = Release|Any CPU + {B54702BF-ECAD-4F96-BC43-F5B670F0A668}.Release|x64.ActiveCfg = Release|Any CPU + {B54702BF-ECAD-4F96-BC43-F5B670F0A668}.Release|x64.Build.0 = Release|Any CPU + {B54702BF-ECAD-4F96-BC43-F5B670F0A668}.Release|x86.ActiveCfg = Release|Any CPU + {B54702BF-ECAD-4F96-BC43-F5B670F0A668}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -432,6 +866,7 @@ Global {71C2BC96-C104-4748-B6CF-E74BAC4E709D} = {4827B3EC-73D8-436D-AE2A-5E29AC95FD0C} {21649885-B40A-4F6B-9286-A7E378276C3E} = {C526E8AB-739A-48D7-8FC4-048978C9B650} {DA8E9EC2-990C-4DA2-A3F3-7E7796C71DD8} = {C526E8AB-739A-48D7-8FC4-048978C9B650} + {B54702BF-ECAD-4F96-BC43-F5B670F0A668} = {95A69671-16CA-4133-981C-CC381B7AAA30} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {8CBB7278-D093-448E-B3DE-B5991209A1AA} diff --git a/sandbox/SubCleanDispose/Program.cs b/sandbox/SubCleanDispose/Program.cs new file mode 100644 index 000000000..bce47ffa3 --- /dev/null +++ b/sandbox/SubCleanDispose/Program.cs @@ -0,0 +1,227 @@ +// Manual dispose-race repro for NATS subscriptions and JetStream consumers. +// +// Each scenario publishes N messages with sequential payloads, starts a +// subscriber/consumer, cancels mid-flight, then checks the received set for +// gaps. A gap (received {1, 2, 4} with no 3) means a message was dropped +// during dispose. +// +// Usage: SubCleanDispose [iterations] +// NATS_URL env var overrides the server URL (default nats://127.0.0.1:4222). +using System.Diagnostics; +using NATS.Client.Core; +using NATS.Client.JetStream; +using NATS.Client.JetStream.Models; + +var url = Environment.GetEnvironmentVariable("NATS_URL") ?? "nats://127.0.0.1:4222"; +var iterations = args.Length > 0 && int.TryParse(args[0], out var n) ? n : 20; + +Console.WriteLine($"url={url} iterations={iterations}"); +Console.WriteLine(); + +var failures = new Dictionary +{ + ["core-sub"] = 0, + ["js-fetch"] = 0, + ["js-consume"] = 0, + ["js-ordered"] = 0, +}; + +for (var i = 1; i <= iterations; i++) +{ + await RunOnce("core-sub", i, () => TestCoreSub(url)); + await RunOnce("js-fetch", i, () => TestJsFetch(url)); + await RunOnce("js-consume", i, () => TestJsConsume(url)); + await RunOnce("js-ordered", i, () => TestJsOrdered(url)); +} + +Console.WriteLine(); +Console.WriteLine("Summary:"); +foreach (var (name, count) in failures) + Console.WriteLine($" {name,-12} {count} failure(s)"); + +return failures.Values.Sum() == 0 ? 0 : 1; + +async Task RunOnce(string name, int iter, Func> body) +{ + var sw = Stopwatch.StartNew(); + Result r; + try + { + r = await body(); + } + catch (Exception e) + { + Console.WriteLine($"[{name}] iter {iter} ERROR {e.GetType().Name}: {e.Message}"); + failures[name]++; + return; + } + + sw.Stop(); + var ok = r.Gap == 0; + if (!ok) + failures[name]++; + Console.WriteLine( + $"[{name}] iter {iter,3} published={r.Published,5} received={r.Received,5} highest={r.Highest,5} gap={r.Gap,4} {sw.ElapsedMilliseconds,5}ms {(ok ? "ok" : "DROP")}"); +} + +static async Task TestCoreSub(string url) +{ + const int published = 1000; + await using var nats = new NatsConnection(new NatsOpts { Url = url }); + await nats.ConnectAsync(); + + var subject = $"t.{Guid.NewGuid():N}"; + var received = new HashSet(); + + await using var sub = await nats.SubscribeCoreAsync(subject); + await nats.PingAsync(); + + var readerDone = new TaskCompletionSource(); + var reader = Task.Run(async () => + { + try + { + await foreach (var msg in sub.Msgs.ReadAllAsync()) + { + received.Add(msg.Data); + if (received.Count == 10) + readerDone.TrySetResult(); + } + } + catch (Exception) + { + } + }); + + for (var i = 1; i <= published; i++) + await nats.PublishAsync(subject, i); + + await readerDone.Task; + + // Dispose while messages may still be in-flight. Core NATS has no + // server-side ack tracking, so gaps here indicate drop-on-dispose. + await sub.DisposeAsync(); + await reader; + + return Result.From(published, received); +} + +static async Task TestJsFetch(string url) +{ + const int published = 1000; + await using var nats = new NatsConnection(new NatsOpts { Url = url }); + var js = new NatsJSContext(nats); + var id = Guid.NewGuid().ToString("N").Substring(0, 8); + var stream = $"s_{id}"; + var subject = $"s_{id}.x"; + var consumer = $"c_{id}"; + + await js.CreateStreamAsync(new StreamConfig(stream, new[] { subject })); + var c = await js.CreateOrUpdateConsumerAsync(stream, new ConsumerConfig(consumer) { AckPolicy = ConsumerConfigAckPolicy.Explicit }); + + for (var i = 1; i <= published; i++) + await js.PublishAsync(subject, i); + + var received = new HashSet(); + using var cts = new CancellationTokenSource(); + try + { + await foreach (var m in c.FetchAsync(new NatsJSFetchOpts { MaxMsgs = published, Expires = TimeSpan.FromSeconds(10) }, cancellationToken: cts.Token)) + { + received.Add(m.Data); + await m.AckAsync(); + if (received.Count == 10) + cts.CancelAfter(TimeSpan.FromMilliseconds(1)); + } + } + catch (OperationCanceledException) + { + } + + await js.DeleteStreamAsync(stream); + return Result.From(published, received); +} + +static async Task TestJsConsume(string url) +{ + const int published = 1000; + await using var nats = new NatsConnection(new NatsOpts { Url = url }); + var js = new NatsJSContext(nats); + var id = Guid.NewGuid().ToString("N").Substring(0, 8); + var stream = $"s_{id}"; + var subject = $"s_{id}.x"; + var consumer = $"c_{id}"; + + await js.CreateStreamAsync(new StreamConfig(stream, new[] { subject })); + var c = await js.CreateOrUpdateConsumerAsync(stream, new ConsumerConfig(consumer) { AckPolicy = ConsumerConfigAckPolicy.Explicit }); + + for (var i = 1; i <= published; i++) + await js.PublishAsync(subject, i); + + var received = new HashSet(); + using var cts = new CancellationTokenSource(); + try + { + await foreach (var m in c.ConsumeAsync(opts: new NatsJSConsumeOpts { MaxMsgs = published }, cancellationToken: cts.Token)) + { + received.Add(m.Data); + await m.AckAsync(); + if (received.Count == 10) + cts.CancelAfter(TimeSpan.FromMilliseconds(1)); + } + } + catch (OperationCanceledException) + { + } + + await js.DeleteStreamAsync(stream); + return Result.From(published, received); +} + +static async Task TestJsOrdered(string url) +{ + const int published = 1000; + await using var nats = new NatsConnection(new NatsOpts { Url = url }); + var js = new NatsJSContext(nats); + var id = Guid.NewGuid().ToString("N").Substring(0, 8); + var stream = $"s_{id}"; + var subject = $"s_{id}.x"; + + await js.CreateStreamAsync(new StreamConfig(stream, new[] { subject })); + var c = await js.CreateOrderedConsumerAsync(stream, new NatsJSOrderedConsumerOpts { FilterSubjects = new[] { subject } }); + + for (var i = 1; i <= published; i++) + await js.PublishAsync(subject, i); + + var received = new HashSet(); + using var cts = new CancellationTokenSource(); + try + { + await foreach (var m in c.ConsumeAsync(cancellationToken: cts.Token)) + { + received.Add(m.Data); + if (received.Count == 10) + cts.CancelAfter(TimeSpan.FromMilliseconds(1)); + } + } + catch (OperationCanceledException) + { + } + + await js.DeleteStreamAsync(stream); + return Result.From(published, received); +} + +internal readonly record struct Result(int Published, int Received, int Highest, int Gap) +{ + public static Result From(int published, HashSet received) + { + if (received.Count == 0) + return new Result(published, 0, 0, 0); + var max = received.Max(); + + // Gap = "how many integers in [1..max] are missing from received". + var gap = max - received.Count; + return new Result(published, received.Count, max, gap); + } +} diff --git a/sandbox/SubCleanDispose/SubCleanDispose.csproj b/sandbox/SubCleanDispose/SubCleanDispose.csproj new file mode 100644 index 000000000..befc5a0df --- /dev/null +++ b/sandbox/SubCleanDispose/SubCleanDispose.csproj @@ -0,0 +1,16 @@ + + + + Exe + net8.0 + enable + enable + false + + + + + + + + diff --git a/src/NATS.Client.Core/NatsSubBase.cs b/src/NATS.Client.Core/NatsSubBase.cs index 9dc02b01a..23fdc6387 100644 --- a/src/NATS.Client.Core/NatsSubBase.cs +++ b/src/NATS.Client.Core/NatsSubBase.cs @@ -395,7 +395,7 @@ internal async ValueTask DrainAsync() _logger.LogWarning(NatsLogEvents.Subscription, e, "Error during drain ping"); } - // Now it's safe to complete the channel — no more messages will arrive. + // Now it's safe to complete the channel, no more messages will arrive. try { TryComplete(); diff --git a/src/NATS.Client.JetStream/Internal/NatsJSConsume.cs b/src/NATS.Client.JetStream/Internal/NatsJSConsume.cs index 4a6613ba9..daefaaa14 100644 --- a/src/NATS.Client.JetStream/Internal/NatsJSConsume.cs +++ b/src/NATS.Client.JetStream/Internal/NatsJSConsume.cs @@ -220,7 +220,7 @@ public override async ValueTask DisposeAsync() Interlocked.Exchange(ref _disposed, 1); try { - // Drain: UNSUB → PING/PONG → TryComplete, so all in-flight + // Drain: UNSUB -> PING/PONG -> TryComplete, so all in-flight // messages are written to the channel before it's completed. await DrainAsync().ConfigureAwait(false); await base.DisposeAsync().ConfigureAwait(false); diff --git a/src/NATS.Client.JetStream/Internal/NatsJSFetch.cs b/src/NATS.Client.JetStream/Internal/NatsJSFetch.cs index 22ca45b30..015d0bb58 100644 --- a/src/NATS.Client.JetStream/Internal/NatsJSFetch.cs +++ b/src/NATS.Client.JetStream/Internal/NatsJSFetch.cs @@ -165,7 +165,7 @@ public override async ValueTask DisposeAsync() Interlocked.Exchange(ref _disposed, 1); try { - // Drain: UNSUB → PING/PONG → TryComplete, so all in-flight + // Drain: UNSUB -> PING/PONG -> TryComplete, so all in-flight // messages are written to the channel before it's completed. await DrainAsync().ConfigureAwait(false); await base.DisposeAsync().ConfigureAwait(false); diff --git a/src/NATS.Client.JetStream/Internal/NatsJSOrderedConsume.cs b/src/NATS.Client.JetStream/Internal/NatsJSOrderedConsume.cs index d327b9e3e..09cada970 100644 --- a/src/NATS.Client.JetStream/Internal/NatsJSOrderedConsume.cs +++ b/src/NATS.Client.JetStream/Internal/NatsJSOrderedConsume.cs @@ -139,6 +139,9 @@ public override async ValueTask DisposeAsync() _context.Connection.ConnectionDisconnected -= ConnectionOnConnectionDisconnected; try { + // Drain: UNSUB -> PING/PONG -> TryComplete, so all in-flight + // messages are written to the channel before it's completed. + await DrainAsync().ConfigureAwait(false); await base.DisposeAsync().ConfigureAwait(false); } finally @@ -306,17 +309,12 @@ protected override async ValueTask ReceiveInternalAsync( } } - // Stop feeding the user if we are disposed. - // We need to exit as soon as possible. - if (Volatile.Read(ref _disposed) == 0) - { - // We can't pass cancellation token here because we need to hand - // the message to the user to be processed. Writer will be completed - // when the user calls Stop() or when the subscription is closed. - await _userMsgs.Writer.WriteAsync(msg).ConfigureAwait(false); + // We can't pass cancellation token here because we need to hand + // the message to the user to be processed. Writer will be completed + // when the user calls Stop() or when the subscription is closed. + await _userMsgs.Writer.WriteAsync(msg).ConfigureAwait(false); - ResetSlowConsumer(_userMsgs.Reader.Count); - } + ResetSlowConsumer(_userMsgs.Reader.Count); } CheckPending(); diff --git a/tests/NATS.Client.JetStream.Tests/ConsumerFetchTest.cs b/tests/NATS.Client.JetStream.Tests/ConsumerFetchTest.cs index f3adf802d..cac69272c 100644 --- a/tests/NATS.Client.JetStream.Tests/ConsumerFetchTest.cs +++ b/tests/NATS.Client.JetStream.Tests/ConsumerFetchTest.cs @@ -77,6 +77,12 @@ public async Task FetchNoWait_test(NatsRequestReplyMode mode) Assert.Equal(10, count); } + // Canary for the dispose race in JetStream fetch/consume/ordered-consume: + // messages in-flight when DisposeAsync is called must all reach the + // channel before it completes. If this test starts flapping, check the + // DrainAsync pattern in NatsSubBase and the DisposeAsync overrides in + // NatsJSFetch, NatsJSConsume, NatsJSOrderedConsume. A manual stress + // repro lives under sandbox/SubCleanDispose. [Theory] // TODO: Fix this test @@ -87,8 +93,7 @@ public async Task Fetch_dispose_test(NatsRequestReplyMode mode) await using var nats = new NatsConnection(new NatsOpts { Url = _server.Url, RequestReplyMode = mode }); var prefix = _server.GetNextId(); - // var cts = new CancellationTokenSource(TimeSpan.FromSeconds(60)); - var cts = new CancellationTokenSource(); + var cts = new CancellationTokenSource(TimeSpan.FromSeconds(60)); var js = new NatsJSContext(nats); @@ -166,121 +171,4 @@ await Retry.Until( await consumer.RefreshAsync(cts.Token); Assert.Equal(0, consumer.Info.NumAckPending); } - - [Fact] - public async Task Fetch_dispose_does_not_drop_buffered_messages() - { - // Reproduce the race: messages delivered by server but dropped during - // dispose because the channel is completed before all writes finish. - // Strategy: pump thousands of messages so there are always messages - // in-flight during dispose, then check server stats for the gap. - // Run iterations in parallel (4x CPU count) to maximize contention. - var iterations = Environment.ProcessorCount * 4; - - await using var nats = new NatsConnection(new NatsOpts { Url = _server.Url }); - var prefix = _server.GetNextId(); - var cts = new CancellationTokenSource(TimeSpan.FromSeconds(120)); - - var js = new NatsJSContext(nats); - var streamName = $"{prefix}s1"; - var subject = $"{prefix}s1.foo"; - - await js.CreateStreamAsync(streamName, new[] { $"{prefix}s1.*" }, cts.Token); - - // Keep publishing messages in the background throughout the test - // to ensure there are always messages in-flight during dispose. - var publishTask = Task.Run( - async () => - { - var seq = 0; - while (!cts.Token.IsCancellationRequested) - { - for (var i = 0; i < 10_000; i++) - { - var ack = await js.PublishAsync( - subject, - new TestData { Test = seq++ }, - serializer: TestDataJsonSerializer.Default, - cancellationToken: cts.Token); - ack.EnsureSuccess(); - } - - await Task.Delay(TimeSpan.FromMilliseconds(10), cts.Token).ConfigureAwait(false); - } - }, - cts.Token); - - // Wait for first batch to land before starting consumers. - await Retry.Until( - "initial messages published", - async () => - { - var stream = await js.GetStreamAsync(streamName, cancellationToken: cts.Token); - return stream.Info.State.Messages >= 10_000; - }, - timeout: TimeSpan.FromSeconds(30)); - - var tasks = Enumerable.Range(0, iterations).Select(iteration => Task.Run(async () => - { - var consumerName = $"{prefix}c{iteration}"; - var consumer = (NatsJSConsumer)await js.CreateOrUpdateConsumerAsync( - streamName, - consumerName, - cancellationToken: cts.Token); - - var fetchOpts = new NatsJSFetchOpts - { - MaxMsgs = 100_000, - IdleHeartbeat = TimeSpan.FromSeconds(5), - Expires = TimeSpan.FromSeconds(30), - }; - - var fc = await consumer.FetchInternalAsync( - serializer: TestDataJsonSerializer.Default, - opts: fetchOpts, - cancellationToken: cts.Token); - - // Wait until some messages arrive but not all — we want - // messages still in-flight when we dispose. - await Retry.Until( - "some messages in channel", - () => fc.Msgs.Count >= 10, - timeout: TimeSpan.FromSeconds(30)); - - // Dispose while messages are still being delivered. - await fc.DisposeAsync(); - - // Read and ack everything that made it into the channel. - var received = 0; - await foreach (var msg in fc.Msgs.ReadAllAsync(cts.Token)) - { - received++; - await msg.AckAsync(cancellationToken: cts.Token); - } - - // Check server-side: delivered - acked should be zero. - // If messages were delivered but dropped before reaching the - // channel, NumAckPending > 0 (server sent them, we never acked). - await consumer.RefreshAsync(cts.Token); - var delivered = consumer.Info.Delivered.ConsumerSeq; - var ackFloor = consumer.Info.AckFloor.ConsumerSeq; - var ackPending = consumer.Info.NumAckPending; - - _output.WriteLine($"iter:{iteration} received:{received} delivered:{delivered} ackFloor:{ackFloor} ackPending:{ackPending}"); - - Assert.Equal(0, ackPending); - })).ToArray(); - - await Task.WhenAll(tasks); - - // Stop the background publisher. - cts.Cancel(); - try - { - await publishTask; - } - catch (OperationCanceledException) - { - } - } } From 6b770a9267e70a149edf11ecd18a660ffda9aa86 Mon Sep 17 00:00:00 2001 From: Ziya Suzen Date: Sun, 26 Apr 2026 14:32:15 +0100 Subject: [PATCH 05/19] Wait for consumer reader to drain on dispose Disposing the connection mid-consume tore down CommandWriter before the loop could ack buffered messages, throwing ObjectDisposedException and leaving messages stuck NumAckPending until AckWait expired. Reorder NatsConnection.DisposeAsync so subscription drain and writer flush run before the socket closes, and add a reader-active flag so DrainAsync waits for the user iterator's finally to run. Drop the SubCleanDispose sandbox; the new test in ConsumerConsumeTest guards the regression. --- NATS.Net.sln | 435 ------------------ sandbox/SubCleanDispose/Program.cs | 227 --------- .../SubCleanDispose/SubCleanDispose.csproj | 16 - src/NATS.Client.Core/NatsConnection.cs | 4 +- src/NATS.Client.Core/NatsSubBase.cs | 11 + .../Internal/NatsJSConsume.cs | 26 ++ .../Internal/NatsJSFetch.cs | 26 ++ .../Internal/NatsJSOrderedConsume.cs | 26 ++ src/NATS.Client.JetStream/NatsJSConsumer.cs | 180 +++++--- .../NatsJSOrderedConsumer.cs | 118 ++--- .../ConsumerConsumeTest.cs | 73 +++ .../ConsumerFetchTest.cs | 3 +- 12 files changed, 334 insertions(+), 811 deletions(-) delete mode 100644 sandbox/SubCleanDispose/Program.cs delete mode 100644 sandbox/SubCleanDispose/SubCleanDispose.csproj diff --git a/NATS.Net.sln b/NATS.Net.sln index 4703b5ab5..0cd496046 100644 --- a/NATS.Net.sln +++ b/NATS.Net.sln @@ -158,654 +158,220 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NATS.Client.CheckAbiTransie EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NATS.Client.CheckAbi", "tests\NATS.Client.CheckAbi\NATS.Client.CheckAbi.csproj", "{DA8E9EC2-990C-4DA2-A3F3-7E7796C71DD8}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SubCleanDispose", "sandbox\SubCleanDispose\SubCleanDispose.csproj", "{B54702BF-ECAD-4F96-BC43-F5B670F0A668}" -EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU - Debug|x64 = Debug|x64 - Debug|x86 = Debug|x86 Release|Any CPU = Release|Any CPU - Release|x64 = Release|x64 - Release|x86 = Release|x86 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution {702B4B02-7FEC-42A1-A399-6F39D4E2CA29}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {702B4B02-7FEC-42A1-A399-6F39D4E2CA29}.Debug|Any CPU.Build.0 = Debug|Any CPU - {702B4B02-7FEC-42A1-A399-6F39D4E2CA29}.Debug|x64.ActiveCfg = Debug|Any CPU - {702B4B02-7FEC-42A1-A399-6F39D4E2CA29}.Debug|x64.Build.0 = Debug|Any CPU - {702B4B02-7FEC-42A1-A399-6F39D4E2CA29}.Debug|x86.ActiveCfg = Debug|Any CPU - {702B4B02-7FEC-42A1-A399-6F39D4E2CA29}.Debug|x86.Build.0 = Debug|Any CPU {702B4B02-7FEC-42A1-A399-6F39D4E2CA29}.Release|Any CPU.ActiveCfg = Release|Any CPU {702B4B02-7FEC-42A1-A399-6F39D4E2CA29}.Release|Any CPU.Build.0 = Release|Any CPU - {702B4B02-7FEC-42A1-A399-6F39D4E2CA29}.Release|x64.ActiveCfg = Release|Any CPU - {702B4B02-7FEC-42A1-A399-6F39D4E2CA29}.Release|x64.Build.0 = Release|Any CPU - {702B4B02-7FEC-42A1-A399-6F39D4E2CA29}.Release|x86.ActiveCfg = Release|Any CPU - {702B4B02-7FEC-42A1-A399-6F39D4E2CA29}.Release|x86.Build.0 = Release|Any CPU {7FE6D43B-181E-485E-8C14-35EAA0EBF09A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {7FE6D43B-181E-485E-8C14-35EAA0EBF09A}.Debug|Any CPU.Build.0 = Debug|Any CPU - {7FE6D43B-181E-485E-8C14-35EAA0EBF09A}.Debug|x64.ActiveCfg = Debug|Any CPU - {7FE6D43B-181E-485E-8C14-35EAA0EBF09A}.Debug|x64.Build.0 = Debug|Any CPU - {7FE6D43B-181E-485E-8C14-35EAA0EBF09A}.Debug|x86.ActiveCfg = Debug|Any CPU - {7FE6D43B-181E-485E-8C14-35EAA0EBF09A}.Debug|x86.Build.0 = Debug|Any CPU {7FE6D43B-181E-485E-8C14-35EAA0EBF09A}.Release|Any CPU.ActiveCfg = Release|Any CPU {7FE6D43B-181E-485E-8C14-35EAA0EBF09A}.Release|Any CPU.Build.0 = Release|Any CPU - {7FE6D43B-181E-485E-8C14-35EAA0EBF09A}.Release|x64.ActiveCfg = Release|Any CPU - {7FE6D43B-181E-485E-8C14-35EAA0EBF09A}.Release|x64.Build.0 = Release|Any CPU - {7FE6D43B-181E-485E-8C14-35EAA0EBF09A}.Release|x86.ActiveCfg = Release|Any CPU - {7FE6D43B-181E-485E-8C14-35EAA0EBF09A}.Release|x86.Build.0 = Release|Any CPU {D60B2ADF-450D-4F8F-A3C5-7803D36FE06B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {D60B2ADF-450D-4F8F-A3C5-7803D36FE06B}.Debug|Any CPU.Build.0 = Debug|Any CPU - {D60B2ADF-450D-4F8F-A3C5-7803D36FE06B}.Debug|x64.ActiveCfg = Debug|Any CPU - {D60B2ADF-450D-4F8F-A3C5-7803D36FE06B}.Debug|x64.Build.0 = Debug|Any CPU - {D60B2ADF-450D-4F8F-A3C5-7803D36FE06B}.Debug|x86.ActiveCfg = Debug|Any CPU - {D60B2ADF-450D-4F8F-A3C5-7803D36FE06B}.Debug|x86.Build.0 = Debug|Any CPU {D60B2ADF-450D-4F8F-A3C5-7803D36FE06B}.Release|Any CPU.ActiveCfg = Release|Any CPU {D60B2ADF-450D-4F8F-A3C5-7803D36FE06B}.Release|Any CPU.Build.0 = Release|Any CPU - {D60B2ADF-450D-4F8F-A3C5-7803D36FE06B}.Release|x64.ActiveCfg = Release|Any CPU - {D60B2ADF-450D-4F8F-A3C5-7803D36FE06B}.Release|x64.Build.0 = Release|Any CPU - {D60B2ADF-450D-4F8F-A3C5-7803D36FE06B}.Release|x86.ActiveCfg = Release|Any CPU - {D60B2ADF-450D-4F8F-A3C5-7803D36FE06B}.Release|x86.Build.0 = Release|Any CPU {A10F0D89-13F3-49B3-ACF7-66E45DC95225}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {A10F0D89-13F3-49B3-ACF7-66E45DC95225}.Debug|Any CPU.Build.0 = Debug|Any CPU - {A10F0D89-13F3-49B3-ACF7-66E45DC95225}.Debug|x64.ActiveCfg = Debug|Any CPU - {A10F0D89-13F3-49B3-ACF7-66E45DC95225}.Debug|x64.Build.0 = Debug|Any CPU - {A10F0D89-13F3-49B3-ACF7-66E45DC95225}.Debug|x86.ActiveCfg = Debug|Any CPU - {A10F0D89-13F3-49B3-ACF7-66E45DC95225}.Debug|x86.Build.0 = Debug|Any CPU {A10F0D89-13F3-49B3-ACF7-66E45DC95225}.Release|Any CPU.ActiveCfg = Release|Any CPU {A10F0D89-13F3-49B3-ACF7-66E45DC95225}.Release|Any CPU.Build.0 = Release|Any CPU - {A10F0D89-13F3-49B3-ACF7-66E45DC95225}.Release|x64.ActiveCfg = Release|Any CPU - {A10F0D89-13F3-49B3-ACF7-66E45DC95225}.Release|x64.Build.0 = Release|Any CPU - {A10F0D89-13F3-49B3-ACF7-66E45DC95225}.Release|x86.ActiveCfg = Release|Any CPU - {A10F0D89-13F3-49B3-ACF7-66E45DC95225}.Release|x86.Build.0 = Release|Any CPU {D3F09B30-1ED5-48C2-81CD-A2AD88E751AC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {D3F09B30-1ED5-48C2-81CD-A2AD88E751AC}.Debug|Any CPU.Build.0 = Debug|Any CPU - {D3F09B30-1ED5-48C2-81CD-A2AD88E751AC}.Debug|x64.ActiveCfg = Debug|Any CPU - {D3F09B30-1ED5-48C2-81CD-A2AD88E751AC}.Debug|x64.Build.0 = Debug|Any CPU - {D3F09B30-1ED5-48C2-81CD-A2AD88E751AC}.Debug|x86.ActiveCfg = Debug|Any CPU - {D3F09B30-1ED5-48C2-81CD-A2AD88E751AC}.Debug|x86.Build.0 = Debug|Any CPU {D3F09B30-1ED5-48C2-81CD-A2AD88E751AC}.Release|Any CPU.ActiveCfg = Release|Any CPU {D3F09B30-1ED5-48C2-81CD-A2AD88E751AC}.Release|Any CPU.Build.0 = Release|Any CPU - {D3F09B30-1ED5-48C2-81CD-A2AD88E751AC}.Release|x64.ActiveCfg = Release|Any CPU - {D3F09B30-1ED5-48C2-81CD-A2AD88E751AC}.Release|x64.Build.0 = Release|Any CPU - {D3F09B30-1ED5-48C2-81CD-A2AD88E751AC}.Release|x86.ActiveCfg = Release|Any CPU - {D3F09B30-1ED5-48C2-81CD-A2AD88E751AC}.Release|x86.Build.0 = Release|Any CPU {44881DEE-8B49-44EA-B0BA-8BDA4F706E1A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {44881DEE-8B49-44EA-B0BA-8BDA4F706E1A}.Debug|Any CPU.Build.0 = Debug|Any CPU - {44881DEE-8B49-44EA-B0BA-8BDA4F706E1A}.Debug|x64.ActiveCfg = Debug|Any CPU - {44881DEE-8B49-44EA-B0BA-8BDA4F706E1A}.Debug|x64.Build.0 = Debug|Any CPU - {44881DEE-8B49-44EA-B0BA-8BDA4F706E1A}.Debug|x86.ActiveCfg = Debug|Any CPU - {44881DEE-8B49-44EA-B0BA-8BDA4F706E1A}.Debug|x86.Build.0 = Debug|Any CPU {44881DEE-8B49-44EA-B0BA-8BDA4F706E1A}.Release|Any CPU.ActiveCfg = Release|Any CPU {44881DEE-8B49-44EA-B0BA-8BDA4F706E1A}.Release|Any CPU.Build.0 = Release|Any CPU - {44881DEE-8B49-44EA-B0BA-8BDA4F706E1A}.Release|x64.ActiveCfg = Release|Any CPU - {44881DEE-8B49-44EA-B0BA-8BDA4F706E1A}.Release|x64.Build.0 = Release|Any CPU - {44881DEE-8B49-44EA-B0BA-8BDA4F706E1A}.Release|x86.ActiveCfg = Release|Any CPU - {44881DEE-8B49-44EA-B0BA-8BDA4F706E1A}.Release|x86.Build.0 = Release|Any CPU {56A9B885-6B7E-4BC6-AED0-ADC67C9A68DB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {56A9B885-6B7E-4BC6-AED0-ADC67C9A68DB}.Debug|Any CPU.Build.0 = Debug|Any CPU - {56A9B885-6B7E-4BC6-AED0-ADC67C9A68DB}.Debug|x64.ActiveCfg = Debug|Any CPU - {56A9B885-6B7E-4BC6-AED0-ADC67C9A68DB}.Debug|x64.Build.0 = Debug|Any CPU - {56A9B885-6B7E-4BC6-AED0-ADC67C9A68DB}.Debug|x86.ActiveCfg = Debug|Any CPU - {56A9B885-6B7E-4BC6-AED0-ADC67C9A68DB}.Debug|x86.Build.0 = Debug|Any CPU {56A9B885-6B7E-4BC6-AED0-ADC67C9A68DB}.Release|Any CPU.ActiveCfg = Release|Any CPU {56A9B885-6B7E-4BC6-AED0-ADC67C9A68DB}.Release|Any CPU.Build.0 = Release|Any CPU - {56A9B885-6B7E-4BC6-AED0-ADC67C9A68DB}.Release|x64.ActiveCfg = Release|Any CPU - {56A9B885-6B7E-4BC6-AED0-ADC67C9A68DB}.Release|x64.Build.0 = Release|Any CPU - {56A9B885-6B7E-4BC6-AED0-ADC67C9A68DB}.Release|x86.ActiveCfg = Release|Any CPU - {56A9B885-6B7E-4BC6-AED0-ADC67C9A68DB}.Release|x86.Build.0 = Release|Any CPU {2D39F649-C512-4EE5-9DFA-7BD4D9E4F145}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {2D39F649-C512-4EE5-9DFA-7BD4D9E4F145}.Debug|Any CPU.Build.0 = Debug|Any CPU - {2D39F649-C512-4EE5-9DFA-7BD4D9E4F145}.Debug|x64.ActiveCfg = Debug|Any CPU - {2D39F649-C512-4EE5-9DFA-7BD4D9E4F145}.Debug|x64.Build.0 = Debug|Any CPU - {2D39F649-C512-4EE5-9DFA-7BD4D9E4F145}.Debug|x86.ActiveCfg = Debug|Any CPU - {2D39F649-C512-4EE5-9DFA-7BD4D9E4F145}.Debug|x86.Build.0 = Debug|Any CPU {2D39F649-C512-4EE5-9DFA-7BD4D9E4F145}.Release|Any CPU.ActiveCfg = Release|Any CPU {2D39F649-C512-4EE5-9DFA-7BD4D9E4F145}.Release|Any CPU.Build.0 = Release|Any CPU - {2D39F649-C512-4EE5-9DFA-7BD4D9E4F145}.Release|x64.ActiveCfg = Release|Any CPU - {2D39F649-C512-4EE5-9DFA-7BD4D9E4F145}.Release|x64.Build.0 = Release|Any CPU - {2D39F649-C512-4EE5-9DFA-7BD4D9E4F145}.Release|x86.ActiveCfg = Release|Any CPU - {2D39F649-C512-4EE5-9DFA-7BD4D9E4F145}.Release|x86.Build.0 = Release|Any CPU {90E5BF38-70C1-460A-9177-CE42815BDBF5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {90E5BF38-70C1-460A-9177-CE42815BDBF5}.Debug|Any CPU.Build.0 = Debug|Any CPU - {90E5BF38-70C1-460A-9177-CE42815BDBF5}.Debug|x64.ActiveCfg = Debug|Any CPU - {90E5BF38-70C1-460A-9177-CE42815BDBF5}.Debug|x64.Build.0 = Debug|Any CPU - {90E5BF38-70C1-460A-9177-CE42815BDBF5}.Debug|x86.ActiveCfg = Debug|Any CPU - {90E5BF38-70C1-460A-9177-CE42815BDBF5}.Debug|x86.Build.0 = Debug|Any CPU {90E5BF38-70C1-460A-9177-CE42815BDBF5}.Release|Any CPU.ActiveCfg = Release|Any CPU {90E5BF38-70C1-460A-9177-CE42815BDBF5}.Release|Any CPU.Build.0 = Release|Any CPU - {90E5BF38-70C1-460A-9177-CE42815BDBF5}.Release|x64.ActiveCfg = Release|Any CPU - {90E5BF38-70C1-460A-9177-CE42815BDBF5}.Release|x64.Build.0 = Release|Any CPU - {90E5BF38-70C1-460A-9177-CE42815BDBF5}.Release|x86.ActiveCfg = Release|Any CPU - {90E5BF38-70C1-460A-9177-CE42815BDBF5}.Release|x86.Build.0 = Release|Any CPU {B7DD4A9C-2D24-4772-951E-86A665C59ADF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {B7DD4A9C-2D24-4772-951E-86A665C59ADF}.Debug|Any CPU.Build.0 = Debug|Any CPU - {B7DD4A9C-2D24-4772-951E-86A665C59ADF}.Debug|x64.ActiveCfg = Debug|Any CPU - {B7DD4A9C-2D24-4772-951E-86A665C59ADF}.Debug|x64.Build.0 = Debug|Any CPU - {B7DD4A9C-2D24-4772-951E-86A665C59ADF}.Debug|x86.ActiveCfg = Debug|Any CPU - {B7DD4A9C-2D24-4772-951E-86A665C59ADF}.Debug|x86.Build.0 = Debug|Any CPU {B7DD4A9C-2D24-4772-951E-86A665C59ADF}.Release|Any CPU.ActiveCfg = Release|Any CPU {B7DD4A9C-2D24-4772-951E-86A665C59ADF}.Release|Any CPU.Build.0 = Release|Any CPU - {B7DD4A9C-2D24-4772-951E-86A665C59ADF}.Release|x64.ActiveCfg = Release|Any CPU - {B7DD4A9C-2D24-4772-951E-86A665C59ADF}.Release|x64.Build.0 = Release|Any CPU - {B7DD4A9C-2D24-4772-951E-86A665C59ADF}.Release|x86.ActiveCfg = Release|Any CPU - {B7DD4A9C-2D24-4772-951E-86A665C59ADF}.Release|x86.Build.0 = Release|Any CPU {ADF66CBA-4F3E-4E91-9842-E194E3BC06A1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {ADF66CBA-4F3E-4E91-9842-E194E3BC06A1}.Debug|Any CPU.Build.0 = Debug|Any CPU - {ADF66CBA-4F3E-4E91-9842-E194E3BC06A1}.Debug|x64.ActiveCfg = Debug|Any CPU - {ADF66CBA-4F3E-4E91-9842-E194E3BC06A1}.Debug|x64.Build.0 = Debug|Any CPU - {ADF66CBA-4F3E-4E91-9842-E194E3BC06A1}.Debug|x86.ActiveCfg = Debug|Any CPU - {ADF66CBA-4F3E-4E91-9842-E194E3BC06A1}.Debug|x86.Build.0 = Debug|Any CPU {ADF66CBA-4F3E-4E91-9842-E194E3BC06A1}.Release|Any CPU.ActiveCfg = Release|Any CPU {ADF66CBA-4F3E-4E91-9842-E194E3BC06A1}.Release|Any CPU.Build.0 = Release|Any CPU - {ADF66CBA-4F3E-4E91-9842-E194E3BC06A1}.Release|x64.ActiveCfg = Release|Any CPU - {ADF66CBA-4F3E-4E91-9842-E194E3BC06A1}.Release|x64.Build.0 = Release|Any CPU - {ADF66CBA-4F3E-4E91-9842-E194E3BC06A1}.Release|x86.ActiveCfg = Release|Any CPU - {ADF66CBA-4F3E-4E91-9842-E194E3BC06A1}.Release|x86.Build.0 = Release|Any CPU {A102AB7B-A90C-4717-B17C-045240838060}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {A102AB7B-A90C-4717-B17C-045240838060}.Debug|Any CPU.Build.0 = Debug|Any CPU - {A102AB7B-A90C-4717-B17C-045240838060}.Debug|x64.ActiveCfg = Debug|Any CPU - {A102AB7B-A90C-4717-B17C-045240838060}.Debug|x64.Build.0 = Debug|Any CPU - {A102AB7B-A90C-4717-B17C-045240838060}.Debug|x86.ActiveCfg = Debug|Any CPU - {A102AB7B-A90C-4717-B17C-045240838060}.Debug|x86.Build.0 = Debug|Any CPU {A102AB7B-A90C-4717-B17C-045240838060}.Release|Any CPU.ActiveCfg = Release|Any CPU {A102AB7B-A90C-4717-B17C-045240838060}.Release|Any CPU.Build.0 = Release|Any CPU - {A102AB7B-A90C-4717-B17C-045240838060}.Release|x64.ActiveCfg = Release|Any CPU - {A102AB7B-A90C-4717-B17C-045240838060}.Release|x64.Build.0 = Release|Any CPU - {A102AB7B-A90C-4717-B17C-045240838060}.Release|x86.ActiveCfg = Release|Any CPU - {A102AB7B-A90C-4717-B17C-045240838060}.Release|x86.Build.0 = Release|Any CPU {908F2CED-CAC0-4A4E-AD19-362A413B5DA4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {908F2CED-CAC0-4A4E-AD19-362A413B5DA4}.Debug|Any CPU.Build.0 = Debug|Any CPU - {908F2CED-CAC0-4A4E-AD19-362A413B5DA4}.Debug|x64.ActiveCfg = Debug|Any CPU - {908F2CED-CAC0-4A4E-AD19-362A413B5DA4}.Debug|x64.Build.0 = Debug|Any CPU - {908F2CED-CAC0-4A4E-AD19-362A413B5DA4}.Debug|x86.ActiveCfg = Debug|Any CPU - {908F2CED-CAC0-4A4E-AD19-362A413B5DA4}.Debug|x86.Build.0 = Debug|Any CPU {908F2CED-CAC0-4A4E-AD19-362A413B5DA4}.Release|Any CPU.ActiveCfg = Release|Any CPU {908F2CED-CAC0-4A4E-AD19-362A413B5DA4}.Release|Any CPU.Build.0 = Release|Any CPU - {908F2CED-CAC0-4A4E-AD19-362A413B5DA4}.Release|x64.ActiveCfg = Release|Any CPU - {908F2CED-CAC0-4A4E-AD19-362A413B5DA4}.Release|x64.Build.0 = Release|Any CPU - {908F2CED-CAC0-4A4E-AD19-362A413B5DA4}.Release|x86.ActiveCfg = Release|Any CPU - {908F2CED-CAC0-4A4E-AD19-362A413B5DA4}.Release|x86.Build.0 = Release|Any CPU {3F8840BA-4F91-4359-AA53-6B26823E7F55}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {3F8840BA-4F91-4359-AA53-6B26823E7F55}.Debug|Any CPU.Build.0 = Debug|Any CPU - {3F8840BA-4F91-4359-AA53-6B26823E7F55}.Debug|x64.ActiveCfg = Debug|Any CPU - {3F8840BA-4F91-4359-AA53-6B26823E7F55}.Debug|x64.Build.0 = Debug|Any CPU - {3F8840BA-4F91-4359-AA53-6B26823E7F55}.Debug|x86.ActiveCfg = Debug|Any CPU - {3F8840BA-4F91-4359-AA53-6B26823E7F55}.Debug|x86.Build.0 = Debug|Any CPU {3F8840BA-4F91-4359-AA53-6B26823E7F55}.Release|Any CPU.ActiveCfg = Release|Any CPU {3F8840BA-4F91-4359-AA53-6B26823E7F55}.Release|Any CPU.Build.0 = Release|Any CPU - {3F8840BA-4F91-4359-AA53-6B26823E7F55}.Release|x64.ActiveCfg = Release|Any CPU - {3F8840BA-4F91-4359-AA53-6B26823E7F55}.Release|x64.Build.0 = Release|Any CPU - {3F8840BA-4F91-4359-AA53-6B26823E7F55}.Release|x86.ActiveCfg = Release|Any CPU - {3F8840BA-4F91-4359-AA53-6B26823E7F55}.Release|x86.Build.0 = Release|Any CPU {BB2F4EEE-1AB3-43F7-B004-6C9B3D52353E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {BB2F4EEE-1AB3-43F7-B004-6C9B3D52353E}.Debug|Any CPU.Build.0 = Debug|Any CPU - {BB2F4EEE-1AB3-43F7-B004-6C9B3D52353E}.Debug|x64.ActiveCfg = Debug|Any CPU - {BB2F4EEE-1AB3-43F7-B004-6C9B3D52353E}.Debug|x64.Build.0 = Debug|Any CPU - {BB2F4EEE-1AB3-43F7-B004-6C9B3D52353E}.Debug|x86.ActiveCfg = Debug|Any CPU - {BB2F4EEE-1AB3-43F7-B004-6C9B3D52353E}.Debug|x86.Build.0 = Debug|Any CPU {BB2F4EEE-1AB3-43F7-B004-6C9B3D52353E}.Release|Any CPU.ActiveCfg = Release|Any CPU {BB2F4EEE-1AB3-43F7-B004-6C9B3D52353E}.Release|Any CPU.Build.0 = Release|Any CPU - {BB2F4EEE-1AB3-43F7-B004-6C9B3D52353E}.Release|x64.ActiveCfg = Release|Any CPU - {BB2F4EEE-1AB3-43F7-B004-6C9B3D52353E}.Release|x64.Build.0 = Release|Any CPU - {BB2F4EEE-1AB3-43F7-B004-6C9B3D52353E}.Release|x86.ActiveCfg = Release|Any CPU - {BB2F4EEE-1AB3-43F7-B004-6C9B3D52353E}.Release|x86.Build.0 = Release|Any CPU {35578296-FF6C-4BA4-BEE5-C1A6E7DB7024}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {35578296-FF6C-4BA4-BEE5-C1A6E7DB7024}.Debug|Any CPU.Build.0 = Debug|Any CPU - {35578296-FF6C-4BA4-BEE5-C1A6E7DB7024}.Debug|x64.ActiveCfg = Debug|Any CPU - {35578296-FF6C-4BA4-BEE5-C1A6E7DB7024}.Debug|x64.Build.0 = Debug|Any CPU - {35578296-FF6C-4BA4-BEE5-C1A6E7DB7024}.Debug|x86.ActiveCfg = Debug|Any CPU - {35578296-FF6C-4BA4-BEE5-C1A6E7DB7024}.Debug|x86.Build.0 = Debug|Any CPU {35578296-FF6C-4BA4-BEE5-C1A6E7DB7024}.Release|Any CPU.ActiveCfg = Release|Any CPU {35578296-FF6C-4BA4-BEE5-C1A6E7DB7024}.Release|Any CPU.Build.0 = Release|Any CPU - {35578296-FF6C-4BA4-BEE5-C1A6E7DB7024}.Release|x64.ActiveCfg = Release|Any CPU - {35578296-FF6C-4BA4-BEE5-C1A6E7DB7024}.Release|x64.Build.0 = Release|Any CPU - {35578296-FF6C-4BA4-BEE5-C1A6E7DB7024}.Release|x86.ActiveCfg = Release|Any CPU - {35578296-FF6C-4BA4-BEE5-C1A6E7DB7024}.Release|x86.Build.0 = Release|Any CPU {050C63EE-8F1C-4535-9C6C-E12E62A1FF1D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {050C63EE-8F1C-4535-9C6C-E12E62A1FF1D}.Debug|Any CPU.Build.0 = Debug|Any CPU - {050C63EE-8F1C-4535-9C6C-E12E62A1FF1D}.Debug|x64.ActiveCfg = Debug|Any CPU - {050C63EE-8F1C-4535-9C6C-E12E62A1FF1D}.Debug|x64.Build.0 = Debug|Any CPU - {050C63EE-8F1C-4535-9C6C-E12E62A1FF1D}.Debug|x86.ActiveCfg = Debug|Any CPU - {050C63EE-8F1C-4535-9C6C-E12E62A1FF1D}.Debug|x86.Build.0 = Debug|Any CPU {050C63EE-8F1C-4535-9C6C-E12E62A1FF1D}.Release|Any CPU.ActiveCfg = Release|Any CPU {050C63EE-8F1C-4535-9C6C-E12E62A1FF1D}.Release|Any CPU.Build.0 = Release|Any CPU - {050C63EE-8F1C-4535-9C6C-E12E62A1FF1D}.Release|x64.ActiveCfg = Release|Any CPU - {050C63EE-8F1C-4535-9C6C-E12E62A1FF1D}.Release|x64.Build.0 = Release|Any CPU - {050C63EE-8F1C-4535-9C6C-E12E62A1FF1D}.Release|x86.ActiveCfg = Release|Any CPU - {050C63EE-8F1C-4535-9C6C-E12E62A1FF1D}.Release|x86.Build.0 = Release|Any CPU {749CAE39-4C1E-4627-9E31-A36B987BC453}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {749CAE39-4C1E-4627-9E31-A36B987BC453}.Debug|Any CPU.Build.0 = Debug|Any CPU - {749CAE39-4C1E-4627-9E31-A36B987BC453}.Debug|x64.ActiveCfg = Debug|Any CPU - {749CAE39-4C1E-4627-9E31-A36B987BC453}.Debug|x64.Build.0 = Debug|Any CPU - {749CAE39-4C1E-4627-9E31-A36B987BC453}.Debug|x86.ActiveCfg = Debug|Any CPU - {749CAE39-4C1E-4627-9E31-A36B987BC453}.Debug|x86.Build.0 = Debug|Any CPU {749CAE39-4C1E-4627-9E31-A36B987BC453}.Release|Any CPU.ActiveCfg = Release|Any CPU {749CAE39-4C1E-4627-9E31-A36B987BC453}.Release|Any CPU.Build.0 = Release|Any CPU - {749CAE39-4C1E-4627-9E31-A36B987BC453}.Release|x64.ActiveCfg = Release|Any CPU - {749CAE39-4C1E-4627-9E31-A36B987BC453}.Release|x64.Build.0 = Release|Any CPU - {749CAE39-4C1E-4627-9E31-A36B987BC453}.Release|x86.ActiveCfg = Release|Any CPU - {749CAE39-4C1E-4627-9E31-A36B987BC453}.Release|x86.Build.0 = Release|Any CPU {CF44A42E-C075-4C5C-BE8B-3DF266FC617B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {CF44A42E-C075-4C5C-BE8B-3DF266FC617B}.Debug|Any CPU.Build.0 = Debug|Any CPU - {CF44A42E-C075-4C5C-BE8B-3DF266FC617B}.Debug|x64.ActiveCfg = Debug|Any CPU - {CF44A42E-C075-4C5C-BE8B-3DF266FC617B}.Debug|x64.Build.0 = Debug|Any CPU - {CF44A42E-C075-4C5C-BE8B-3DF266FC617B}.Debug|x86.ActiveCfg = Debug|Any CPU - {CF44A42E-C075-4C5C-BE8B-3DF266FC617B}.Debug|x86.Build.0 = Debug|Any CPU {CF44A42E-C075-4C5C-BE8B-3DF266FC617B}.Release|Any CPU.ActiveCfg = Release|Any CPU {CF44A42E-C075-4C5C-BE8B-3DF266FC617B}.Release|Any CPU.Build.0 = Release|Any CPU - {CF44A42E-C075-4C5C-BE8B-3DF266FC617B}.Release|x64.ActiveCfg = Release|Any CPU - {CF44A42E-C075-4C5C-BE8B-3DF266FC617B}.Release|x64.Build.0 = Release|Any CPU - {CF44A42E-C075-4C5C-BE8B-3DF266FC617B}.Release|x86.ActiveCfg = Release|Any CPU - {CF44A42E-C075-4C5C-BE8B-3DF266FC617B}.Release|x86.Build.0 = Release|Any CPU {5B4B30A5-941B-44F9-98C6-06F0BB2242AB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {5B4B30A5-941B-44F9-98C6-06F0BB2242AB}.Debug|Any CPU.Build.0 = Debug|Any CPU - {5B4B30A5-941B-44F9-98C6-06F0BB2242AB}.Debug|x64.ActiveCfg = Debug|Any CPU - {5B4B30A5-941B-44F9-98C6-06F0BB2242AB}.Debug|x64.Build.0 = Debug|Any CPU - {5B4B30A5-941B-44F9-98C6-06F0BB2242AB}.Debug|x86.ActiveCfg = Debug|Any CPU - {5B4B30A5-941B-44F9-98C6-06F0BB2242AB}.Debug|x86.Build.0 = Debug|Any CPU {5B4B30A5-941B-44F9-98C6-06F0BB2242AB}.Release|Any CPU.ActiveCfg = Release|Any CPU {5B4B30A5-941B-44F9-98C6-06F0BB2242AB}.Release|Any CPU.Build.0 = Release|Any CPU - {5B4B30A5-941B-44F9-98C6-06F0BB2242AB}.Release|x64.ActiveCfg = Release|Any CPU - {5B4B30A5-941B-44F9-98C6-06F0BB2242AB}.Release|x64.Build.0 = Release|Any CPU - {5B4B30A5-941B-44F9-98C6-06F0BB2242AB}.Release|x86.ActiveCfg = Release|Any CPU - {5B4B30A5-941B-44F9-98C6-06F0BB2242AB}.Release|x86.Build.0 = Release|Any CPU {6A7B9B9F-BFA4-4A6D-9006-0AAF597FC6DD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {6A7B9B9F-BFA4-4A6D-9006-0AAF597FC6DD}.Debug|Any CPU.Build.0 = Debug|Any CPU - {6A7B9B9F-BFA4-4A6D-9006-0AAF597FC6DD}.Debug|x64.ActiveCfg = Debug|Any CPU - {6A7B9B9F-BFA4-4A6D-9006-0AAF597FC6DD}.Debug|x64.Build.0 = Debug|Any CPU - {6A7B9B9F-BFA4-4A6D-9006-0AAF597FC6DD}.Debug|x86.ActiveCfg = Debug|Any CPU - {6A7B9B9F-BFA4-4A6D-9006-0AAF597FC6DD}.Debug|x86.Build.0 = Debug|Any CPU {6A7B9B9F-BFA4-4A6D-9006-0AAF597FC6DD}.Release|Any CPU.ActiveCfg = Release|Any CPU {6A7B9B9F-BFA4-4A6D-9006-0AAF597FC6DD}.Release|Any CPU.Build.0 = Release|Any CPU - {6A7B9B9F-BFA4-4A6D-9006-0AAF597FC6DD}.Release|x64.ActiveCfg = Release|Any CPU - {6A7B9B9F-BFA4-4A6D-9006-0AAF597FC6DD}.Release|x64.Build.0 = Release|Any CPU - {6A7B9B9F-BFA4-4A6D-9006-0AAF597FC6DD}.Release|x86.ActiveCfg = Release|Any CPU - {6A7B9B9F-BFA4-4A6D-9006-0AAF597FC6DD}.Release|x86.Build.0 = Release|Any CPU {389C05EB-A0B3-4097-8C1F-4D55818438CC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {389C05EB-A0B3-4097-8C1F-4D55818438CC}.Debug|Any CPU.Build.0 = Debug|Any CPU - {389C05EB-A0B3-4097-8C1F-4D55818438CC}.Debug|x64.ActiveCfg = Debug|Any CPU - {389C05EB-A0B3-4097-8C1F-4D55818438CC}.Debug|x64.Build.0 = Debug|Any CPU - {389C05EB-A0B3-4097-8C1F-4D55818438CC}.Debug|x86.ActiveCfg = Debug|Any CPU - {389C05EB-A0B3-4097-8C1F-4D55818438CC}.Debug|x86.Build.0 = Debug|Any CPU {389C05EB-A0B3-4097-8C1F-4D55818438CC}.Release|Any CPU.ActiveCfg = Release|Any CPU {389C05EB-A0B3-4097-8C1F-4D55818438CC}.Release|Any CPU.Build.0 = Release|Any CPU - {389C05EB-A0B3-4097-8C1F-4D55818438CC}.Release|x64.ActiveCfg = Release|Any CPU - {389C05EB-A0B3-4097-8C1F-4D55818438CC}.Release|x64.Build.0 = Release|Any CPU - {389C05EB-A0B3-4097-8C1F-4D55818438CC}.Release|x86.ActiveCfg = Release|Any CPU - {389C05EB-A0B3-4097-8C1F-4D55818438CC}.Release|x86.Build.0 = Release|Any CPU {766C2486-34C3-4DD1-B31C-540C17C044B0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {766C2486-34C3-4DD1-B31C-540C17C044B0}.Debug|Any CPU.Build.0 = Debug|Any CPU - {766C2486-34C3-4DD1-B31C-540C17C044B0}.Debug|x64.ActiveCfg = Debug|Any CPU - {766C2486-34C3-4DD1-B31C-540C17C044B0}.Debug|x64.Build.0 = Debug|Any CPU - {766C2486-34C3-4DD1-B31C-540C17C044B0}.Debug|x86.ActiveCfg = Debug|Any CPU - {766C2486-34C3-4DD1-B31C-540C17C044B0}.Debug|x86.Build.0 = Debug|Any CPU {766C2486-34C3-4DD1-B31C-540C17C044B0}.Release|Any CPU.ActiveCfg = Release|Any CPU {766C2486-34C3-4DD1-B31C-540C17C044B0}.Release|Any CPU.Build.0 = Release|Any CPU - {766C2486-34C3-4DD1-B31C-540C17C044B0}.Release|x64.ActiveCfg = Release|Any CPU - {766C2486-34C3-4DD1-B31C-540C17C044B0}.Release|x64.Build.0 = Release|Any CPU - {766C2486-34C3-4DD1-B31C-540C17C044B0}.Release|x86.ActiveCfg = Release|Any CPU - {766C2486-34C3-4DD1-B31C-540C17C044B0}.Release|x86.Build.0 = Release|Any CPU {2EA0EB68-1AA4-40AC-8FA2-B51532075567}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {2EA0EB68-1AA4-40AC-8FA2-B51532075567}.Debug|Any CPU.Build.0 = Debug|Any CPU - {2EA0EB68-1AA4-40AC-8FA2-B51532075567}.Debug|x64.ActiveCfg = Debug|Any CPU - {2EA0EB68-1AA4-40AC-8FA2-B51532075567}.Debug|x64.Build.0 = Debug|Any CPU - {2EA0EB68-1AA4-40AC-8FA2-B51532075567}.Debug|x86.ActiveCfg = Debug|Any CPU - {2EA0EB68-1AA4-40AC-8FA2-B51532075567}.Debug|x86.Build.0 = Debug|Any CPU {2EA0EB68-1AA4-40AC-8FA2-B51532075567}.Release|Any CPU.ActiveCfg = Release|Any CPU {2EA0EB68-1AA4-40AC-8FA2-B51532075567}.Release|Any CPU.Build.0 = Release|Any CPU - {2EA0EB68-1AA4-40AC-8FA2-B51532075567}.Release|x64.ActiveCfg = Release|Any CPU - {2EA0EB68-1AA4-40AC-8FA2-B51532075567}.Release|x64.Build.0 = Release|Any CPU - {2EA0EB68-1AA4-40AC-8FA2-B51532075567}.Release|x86.ActiveCfg = Release|Any CPU - {2EA0EB68-1AA4-40AC-8FA2-B51532075567}.Release|x86.Build.0 = Release|Any CPU {B8554582-DE19-41A2-9784-9B27C9F22429}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {B8554582-DE19-41A2-9784-9B27C9F22429}.Debug|Any CPU.Build.0 = Debug|Any CPU - {B8554582-DE19-41A2-9784-9B27C9F22429}.Debug|x64.ActiveCfg = Debug|Any CPU - {B8554582-DE19-41A2-9784-9B27C9F22429}.Debug|x64.Build.0 = Debug|Any CPU - {B8554582-DE19-41A2-9784-9B27C9F22429}.Debug|x86.ActiveCfg = Debug|Any CPU - {B8554582-DE19-41A2-9784-9B27C9F22429}.Debug|x86.Build.0 = Debug|Any CPU {B8554582-DE19-41A2-9784-9B27C9F22429}.Release|Any CPU.ActiveCfg = Release|Any CPU {B8554582-DE19-41A2-9784-9B27C9F22429}.Release|Any CPU.Build.0 = Release|Any CPU - {B8554582-DE19-41A2-9784-9B27C9F22429}.Release|x64.ActiveCfg = Release|Any CPU - {B8554582-DE19-41A2-9784-9B27C9F22429}.Release|x64.Build.0 = Release|Any CPU - {B8554582-DE19-41A2-9784-9B27C9F22429}.Release|x86.ActiveCfg = Release|Any CPU - {B8554582-DE19-41A2-9784-9B27C9F22429}.Release|x86.Build.0 = Release|Any CPU {A37994CC-A23A-415E-8B61-9468C7178A55}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {A37994CC-A23A-415E-8B61-9468C7178A55}.Debug|Any CPU.Build.0 = Debug|Any CPU - {A37994CC-A23A-415E-8B61-9468C7178A55}.Debug|x64.ActiveCfg = Debug|Any CPU - {A37994CC-A23A-415E-8B61-9468C7178A55}.Debug|x64.Build.0 = Debug|Any CPU - {A37994CC-A23A-415E-8B61-9468C7178A55}.Debug|x86.ActiveCfg = Debug|Any CPU - {A37994CC-A23A-415E-8B61-9468C7178A55}.Debug|x86.Build.0 = Debug|Any CPU {A37994CC-A23A-415E-8B61-9468C7178A55}.Release|Any CPU.ActiveCfg = Release|Any CPU {A37994CC-A23A-415E-8B61-9468C7178A55}.Release|Any CPU.Build.0 = Release|Any CPU - {A37994CC-A23A-415E-8B61-9468C7178A55}.Release|x64.ActiveCfg = Release|Any CPU - {A37994CC-A23A-415E-8B61-9468C7178A55}.Release|x64.Build.0 = Release|Any CPU - {A37994CC-A23A-415E-8B61-9468C7178A55}.Release|x86.ActiveCfg = Release|Any CPU - {A37994CC-A23A-415E-8B61-9468C7178A55}.Release|x86.Build.0 = Release|Any CPU {6DAAAA87-8DDF-4E60-81CE-D8900327DE33}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {6DAAAA87-8DDF-4E60-81CE-D8900327DE33}.Debug|Any CPU.Build.0 = Debug|Any CPU - {6DAAAA87-8DDF-4E60-81CE-D8900327DE33}.Debug|x64.ActiveCfg = Debug|Any CPU - {6DAAAA87-8DDF-4E60-81CE-D8900327DE33}.Debug|x64.Build.0 = Debug|Any CPU - {6DAAAA87-8DDF-4E60-81CE-D8900327DE33}.Debug|x86.ActiveCfg = Debug|Any CPU - {6DAAAA87-8DDF-4E60-81CE-D8900327DE33}.Debug|x86.Build.0 = Debug|Any CPU {6DAAAA87-8DDF-4E60-81CE-D8900327DE33}.Release|Any CPU.ActiveCfg = Release|Any CPU {6DAAAA87-8DDF-4E60-81CE-D8900327DE33}.Release|Any CPU.Build.0 = Release|Any CPU - {6DAAAA87-8DDF-4E60-81CE-D8900327DE33}.Release|x64.ActiveCfg = Release|Any CPU - {6DAAAA87-8DDF-4E60-81CE-D8900327DE33}.Release|x64.Build.0 = Release|Any CPU - {6DAAAA87-8DDF-4E60-81CE-D8900327DE33}.Release|x86.ActiveCfg = Release|Any CPU - {6DAAAA87-8DDF-4E60-81CE-D8900327DE33}.Release|x86.Build.0 = Release|Any CPU {227C88B1-0510-4010-B142-C44725578FCD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {227C88B1-0510-4010-B142-C44725578FCD}.Debug|Any CPU.Build.0 = Debug|Any CPU - {227C88B1-0510-4010-B142-C44725578FCD}.Debug|x64.ActiveCfg = Debug|Any CPU - {227C88B1-0510-4010-B142-C44725578FCD}.Debug|x64.Build.0 = Debug|Any CPU - {227C88B1-0510-4010-B142-C44725578FCD}.Debug|x86.ActiveCfg = Debug|Any CPU - {227C88B1-0510-4010-B142-C44725578FCD}.Debug|x86.Build.0 = Debug|Any CPU {227C88B1-0510-4010-B142-C44725578FCD}.Release|Any CPU.ActiveCfg = Release|Any CPU {227C88B1-0510-4010-B142-C44725578FCD}.Release|Any CPU.Build.0 = Release|Any CPU - {227C88B1-0510-4010-B142-C44725578FCD}.Release|x64.ActiveCfg = Release|Any CPU - {227C88B1-0510-4010-B142-C44725578FCD}.Release|x64.Build.0 = Release|Any CPU - {227C88B1-0510-4010-B142-C44725578FCD}.Release|x86.ActiveCfg = Release|Any CPU - {227C88B1-0510-4010-B142-C44725578FCD}.Release|x86.Build.0 = Release|Any CPU {8A676AAA-FEE3-4C18-870A-66E59AD9069F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {8A676AAA-FEE3-4C18-870A-66E59AD9069F}.Debug|Any CPU.Build.0 = Debug|Any CPU - {8A676AAA-FEE3-4C18-870A-66E59AD9069F}.Debug|x64.ActiveCfg = Debug|Any CPU - {8A676AAA-FEE3-4C18-870A-66E59AD9069F}.Debug|x64.Build.0 = Debug|Any CPU - {8A676AAA-FEE3-4C18-870A-66E59AD9069F}.Debug|x86.ActiveCfg = Debug|Any CPU - {8A676AAA-FEE3-4C18-870A-66E59AD9069F}.Debug|x86.Build.0 = Debug|Any CPU {8A676AAA-FEE3-4C18-870A-66E59AD9069F}.Release|Any CPU.ActiveCfg = Release|Any CPU {8A676AAA-FEE3-4C18-870A-66E59AD9069F}.Release|Any CPU.Build.0 = Release|Any CPU - {8A676AAA-FEE3-4C18-870A-66E59AD9069F}.Release|x64.ActiveCfg = Release|Any CPU - {8A676AAA-FEE3-4C18-870A-66E59AD9069F}.Release|x64.Build.0 = Release|Any CPU - {8A676AAA-FEE3-4C18-870A-66E59AD9069F}.Release|x86.ActiveCfg = Release|Any CPU - {8A676AAA-FEE3-4C18-870A-66E59AD9069F}.Release|x86.Build.0 = Release|Any CPU {9521D9E0-642A-4C7E-BD10-372DF235CF62}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {9521D9E0-642A-4C7E-BD10-372DF235CF62}.Debug|Any CPU.Build.0 = Debug|Any CPU - {9521D9E0-642A-4C7E-BD10-372DF235CF62}.Debug|x64.ActiveCfg = Debug|Any CPU - {9521D9E0-642A-4C7E-BD10-372DF235CF62}.Debug|x64.Build.0 = Debug|Any CPU - {9521D9E0-642A-4C7E-BD10-372DF235CF62}.Debug|x86.ActiveCfg = Debug|Any CPU - {9521D9E0-642A-4C7E-BD10-372DF235CF62}.Debug|x86.Build.0 = Debug|Any CPU {9521D9E0-642A-4C7E-BD10-372DF235CF62}.Release|Any CPU.ActiveCfg = Release|Any CPU {9521D9E0-642A-4C7E-BD10-372DF235CF62}.Release|Any CPU.Build.0 = Release|Any CPU - {9521D9E0-642A-4C7E-BD10-372DF235CF62}.Release|x64.ActiveCfg = Release|Any CPU - {9521D9E0-642A-4C7E-BD10-372DF235CF62}.Release|x64.Build.0 = Release|Any CPU - {9521D9E0-642A-4C7E-BD10-372DF235CF62}.Release|x86.ActiveCfg = Release|Any CPU - {9521D9E0-642A-4C7E-BD10-372DF235CF62}.Release|x86.Build.0 = Release|Any CPU {8C1E83E5-852F-447F-95B3-0CD2199F3A66}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {8C1E83E5-852F-447F-95B3-0CD2199F3A66}.Debug|Any CPU.Build.0 = Debug|Any CPU - {8C1E83E5-852F-447F-95B3-0CD2199F3A66}.Debug|x64.ActiveCfg = Debug|Any CPU - {8C1E83E5-852F-447F-95B3-0CD2199F3A66}.Debug|x64.Build.0 = Debug|Any CPU - {8C1E83E5-852F-447F-95B3-0CD2199F3A66}.Debug|x86.ActiveCfg = Debug|Any CPU - {8C1E83E5-852F-447F-95B3-0CD2199F3A66}.Debug|x86.Build.0 = Debug|Any CPU {8C1E83E5-852F-447F-95B3-0CD2199F3A66}.Release|Any CPU.ActiveCfg = Release|Any CPU {8C1E83E5-852F-447F-95B3-0CD2199F3A66}.Release|Any CPU.Build.0 = Release|Any CPU - {8C1E83E5-852F-447F-95B3-0CD2199F3A66}.Release|x64.ActiveCfg = Release|Any CPU - {8C1E83E5-852F-447F-95B3-0CD2199F3A66}.Release|x64.Build.0 = Release|Any CPU - {8C1E83E5-852F-447F-95B3-0CD2199F3A66}.Release|x86.ActiveCfg = Release|Any CPU - {8C1E83E5-852F-447F-95B3-0CD2199F3A66}.Release|x86.Build.0 = Release|Any CPU {D8E29F7F-8F7C-462D-8E5F-7C0DAA08D527}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {D8E29F7F-8F7C-462D-8E5F-7C0DAA08D527}.Debug|Any CPU.Build.0 = Debug|Any CPU - {D8E29F7F-8F7C-462D-8E5F-7C0DAA08D527}.Debug|x64.ActiveCfg = Debug|Any CPU - {D8E29F7F-8F7C-462D-8E5F-7C0DAA08D527}.Debug|x64.Build.0 = Debug|Any CPU - {D8E29F7F-8F7C-462D-8E5F-7C0DAA08D527}.Debug|x86.ActiveCfg = Debug|Any CPU - {D8E29F7F-8F7C-462D-8E5F-7C0DAA08D527}.Debug|x86.Build.0 = Debug|Any CPU {D8E29F7F-8F7C-462D-8E5F-7C0DAA08D527}.Release|Any CPU.ActiveCfg = Release|Any CPU {D8E29F7F-8F7C-462D-8E5F-7C0DAA08D527}.Release|Any CPU.Build.0 = Release|Any CPU - {D8E29F7F-8F7C-462D-8E5F-7C0DAA08D527}.Release|x64.ActiveCfg = Release|Any CPU - {D8E29F7F-8F7C-462D-8E5F-7C0DAA08D527}.Release|x64.Build.0 = Release|Any CPU - {D8E29F7F-8F7C-462D-8E5F-7C0DAA08D527}.Release|x86.ActiveCfg = Release|Any CPU - {D8E29F7F-8F7C-462D-8E5F-7C0DAA08D527}.Release|x86.Build.0 = Release|Any CPU {6C039E97-4AFA-4FB7-B361-C40001A632B2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {6C039E97-4AFA-4FB7-B361-C40001A632B2}.Debug|Any CPU.Build.0 = Debug|Any CPU - {6C039E97-4AFA-4FB7-B361-C40001A632B2}.Debug|x64.ActiveCfg = Debug|Any CPU - {6C039E97-4AFA-4FB7-B361-C40001A632B2}.Debug|x64.Build.0 = Debug|Any CPU - {6C039E97-4AFA-4FB7-B361-C40001A632B2}.Debug|x86.ActiveCfg = Debug|Any CPU - {6C039E97-4AFA-4FB7-B361-C40001A632B2}.Debug|x86.Build.0 = Debug|Any CPU {6C039E97-4AFA-4FB7-B361-C40001A632B2}.Release|Any CPU.ActiveCfg = Release|Any CPU {6C039E97-4AFA-4FB7-B361-C40001A632B2}.Release|Any CPU.Build.0 = Release|Any CPU - {6C039E97-4AFA-4FB7-B361-C40001A632B2}.Release|x64.ActiveCfg = Release|Any CPU - {6C039E97-4AFA-4FB7-B361-C40001A632B2}.Release|x64.Build.0 = Release|Any CPU - {6C039E97-4AFA-4FB7-B361-C40001A632B2}.Release|x86.ActiveCfg = Release|Any CPU - {6C039E97-4AFA-4FB7-B361-C40001A632B2}.Release|x86.Build.0 = Release|Any CPU {1A8222A7-699E-493E-852B-87646FD7C311}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {1A8222A7-699E-493E-852B-87646FD7C311}.Debug|Any CPU.Build.0 = Debug|Any CPU - {1A8222A7-699E-493E-852B-87646FD7C311}.Debug|x64.ActiveCfg = Debug|Any CPU - {1A8222A7-699E-493E-852B-87646FD7C311}.Debug|x64.Build.0 = Debug|Any CPU - {1A8222A7-699E-493E-852B-87646FD7C311}.Debug|x86.ActiveCfg = Debug|Any CPU - {1A8222A7-699E-493E-852B-87646FD7C311}.Debug|x86.Build.0 = Debug|Any CPU {1A8222A7-699E-493E-852B-87646FD7C311}.Release|Any CPU.ActiveCfg = Release|Any CPU {1A8222A7-699E-493E-852B-87646FD7C311}.Release|Any CPU.Build.0 = Release|Any CPU - {1A8222A7-699E-493E-852B-87646FD7C311}.Release|x64.ActiveCfg = Release|Any CPU - {1A8222A7-699E-493E-852B-87646FD7C311}.Release|x64.Build.0 = Release|Any CPU - {1A8222A7-699E-493E-852B-87646FD7C311}.Release|x86.ActiveCfg = Release|Any CPU - {1A8222A7-699E-493E-852B-87646FD7C311}.Release|x86.Build.0 = Release|Any CPU {A7B6B97C-278B-4593-9967-6D791045C0B6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {A7B6B97C-278B-4593-9967-6D791045C0B6}.Debug|Any CPU.Build.0 = Debug|Any CPU - {A7B6B97C-278B-4593-9967-6D791045C0B6}.Debug|x64.ActiveCfg = Debug|Any CPU - {A7B6B97C-278B-4593-9967-6D791045C0B6}.Debug|x64.Build.0 = Debug|Any CPU - {A7B6B97C-278B-4593-9967-6D791045C0B6}.Debug|x86.ActiveCfg = Debug|Any CPU - {A7B6B97C-278B-4593-9967-6D791045C0B6}.Debug|x86.Build.0 = Debug|Any CPU {A7B6B97C-278B-4593-9967-6D791045C0B6}.Release|Any CPU.ActiveCfg = Release|Any CPU {A7B6B97C-278B-4593-9967-6D791045C0B6}.Release|Any CPU.Build.0 = Release|Any CPU - {A7B6B97C-278B-4593-9967-6D791045C0B6}.Release|x64.ActiveCfg = Release|Any CPU - {A7B6B97C-278B-4593-9967-6D791045C0B6}.Release|x64.Build.0 = Release|Any CPU - {A7B6B97C-278B-4593-9967-6D791045C0B6}.Release|x86.ActiveCfg = Release|Any CPU - {A7B6B97C-278B-4593-9967-6D791045C0B6}.Release|x86.Build.0 = Release|Any CPU {94AF0E32-E0F7-417D-A010-0545964641F4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {94AF0E32-E0F7-417D-A010-0545964641F4}.Debug|Any CPU.Build.0 = Debug|Any CPU - {94AF0E32-E0F7-417D-A010-0545964641F4}.Debug|x64.ActiveCfg = Debug|Any CPU - {94AF0E32-E0F7-417D-A010-0545964641F4}.Debug|x64.Build.0 = Debug|Any CPU - {94AF0E32-E0F7-417D-A010-0545964641F4}.Debug|x86.ActiveCfg = Debug|Any CPU - {94AF0E32-E0F7-417D-A010-0545964641F4}.Debug|x86.Build.0 = Debug|Any CPU {94AF0E32-E0F7-417D-A010-0545964641F4}.Release|Any CPU.ActiveCfg = Release|Any CPU {94AF0E32-E0F7-417D-A010-0545964641F4}.Release|Any CPU.Build.0 = Release|Any CPU - {94AF0E32-E0F7-417D-A010-0545964641F4}.Release|x64.ActiveCfg = Release|Any CPU - {94AF0E32-E0F7-417D-A010-0545964641F4}.Release|x64.Build.0 = Release|Any CPU - {94AF0E32-E0F7-417D-A010-0545964641F4}.Release|x86.ActiveCfg = Release|Any CPU - {94AF0E32-E0F7-417D-A010-0545964641F4}.Release|x86.Build.0 = Release|Any CPU {C075FACD-83A3-48F2-9F6E-D181F66FB2EC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {C075FACD-83A3-48F2-9F6E-D181F66FB2EC}.Debug|Any CPU.Build.0 = Debug|Any CPU - {C075FACD-83A3-48F2-9F6E-D181F66FB2EC}.Debug|x64.ActiveCfg = Debug|Any CPU - {C075FACD-83A3-48F2-9F6E-D181F66FB2EC}.Debug|x64.Build.0 = Debug|Any CPU - {C075FACD-83A3-48F2-9F6E-D181F66FB2EC}.Debug|x86.ActiveCfg = Debug|Any CPU - {C075FACD-83A3-48F2-9F6E-D181F66FB2EC}.Debug|x86.Build.0 = Debug|Any CPU {C075FACD-83A3-48F2-9F6E-D181F66FB2EC}.Release|Any CPU.ActiveCfg = Release|Any CPU {C075FACD-83A3-48F2-9F6E-D181F66FB2EC}.Release|Any CPU.Build.0 = Release|Any CPU - {C075FACD-83A3-48F2-9F6E-D181F66FB2EC}.Release|x64.ActiveCfg = Release|Any CPU - {C075FACD-83A3-48F2-9F6E-D181F66FB2EC}.Release|x64.Build.0 = Release|Any CPU - {C075FACD-83A3-48F2-9F6E-D181F66FB2EC}.Release|x86.ActiveCfg = Release|Any CPU - {C075FACD-83A3-48F2-9F6E-D181F66FB2EC}.Release|x86.Build.0 = Release|Any CPU {4501D331-9428-48AC-8C89-52034BF7F06F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {4501D331-9428-48AC-8C89-52034BF7F06F}.Debug|Any CPU.Build.0 = Debug|Any CPU - {4501D331-9428-48AC-8C89-52034BF7F06F}.Debug|x64.ActiveCfg = Debug|Any CPU - {4501D331-9428-48AC-8C89-52034BF7F06F}.Debug|x64.Build.0 = Debug|Any CPU - {4501D331-9428-48AC-8C89-52034BF7F06F}.Debug|x86.ActiveCfg = Debug|Any CPU - {4501D331-9428-48AC-8C89-52034BF7F06F}.Debug|x86.Build.0 = Debug|Any CPU {4501D331-9428-48AC-8C89-52034BF7F06F}.Release|Any CPU.ActiveCfg = Release|Any CPU {4501D331-9428-48AC-8C89-52034BF7F06F}.Release|Any CPU.Build.0 = Release|Any CPU - {4501D331-9428-48AC-8C89-52034BF7F06F}.Release|x64.ActiveCfg = Release|Any CPU - {4501D331-9428-48AC-8C89-52034BF7F06F}.Release|x64.Build.0 = Release|Any CPU - {4501D331-9428-48AC-8C89-52034BF7F06F}.Release|x86.ActiveCfg = Release|Any CPU - {4501D331-9428-48AC-8C89-52034BF7F06F}.Release|x86.Build.0 = Release|Any CPU {92AB528E-2BE4-438F-9C0E-83BB3D967403}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {92AB528E-2BE4-438F-9C0E-83BB3D967403}.Debug|Any CPU.Build.0 = Debug|Any CPU - {92AB528E-2BE4-438F-9C0E-83BB3D967403}.Debug|x64.ActiveCfg = Debug|Any CPU - {92AB528E-2BE4-438F-9C0E-83BB3D967403}.Debug|x64.Build.0 = Debug|Any CPU - {92AB528E-2BE4-438F-9C0E-83BB3D967403}.Debug|x86.ActiveCfg = Debug|Any CPU - {92AB528E-2BE4-438F-9C0E-83BB3D967403}.Debug|x86.Build.0 = Debug|Any CPU {92AB528E-2BE4-438F-9C0E-83BB3D967403}.Release|Any CPU.ActiveCfg = Release|Any CPU {92AB528E-2BE4-438F-9C0E-83BB3D967403}.Release|Any CPU.Build.0 = Release|Any CPU - {92AB528E-2BE4-438F-9C0E-83BB3D967403}.Release|x64.ActiveCfg = Release|Any CPU - {92AB528E-2BE4-438F-9C0E-83BB3D967403}.Release|x64.Build.0 = Release|Any CPU - {92AB528E-2BE4-438F-9C0E-83BB3D967403}.Release|x86.ActiveCfg = Release|Any CPU - {92AB528E-2BE4-438F-9C0E-83BB3D967403}.Release|x86.Build.0 = Release|Any CPU {8E7B180C-7057-4C81-BD0F-D35A74501FCB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {8E7B180C-7057-4C81-BD0F-D35A74501FCB}.Debug|Any CPU.Build.0 = Debug|Any CPU - {8E7B180C-7057-4C81-BD0F-D35A74501FCB}.Debug|x64.ActiveCfg = Debug|Any CPU - {8E7B180C-7057-4C81-BD0F-D35A74501FCB}.Debug|x64.Build.0 = Debug|Any CPU - {8E7B180C-7057-4C81-BD0F-D35A74501FCB}.Debug|x86.ActiveCfg = Debug|Any CPU - {8E7B180C-7057-4C81-BD0F-D35A74501FCB}.Debug|x86.Build.0 = Debug|Any CPU {8E7B180C-7057-4C81-BD0F-D35A74501FCB}.Release|Any CPU.ActiveCfg = Release|Any CPU {8E7B180C-7057-4C81-BD0F-D35A74501FCB}.Release|Any CPU.Build.0 = Release|Any CPU - {8E7B180C-7057-4C81-BD0F-D35A74501FCB}.Release|x64.ActiveCfg = Release|Any CPU - {8E7B180C-7057-4C81-BD0F-D35A74501FCB}.Release|x64.Build.0 = Release|Any CPU - {8E7B180C-7057-4C81-BD0F-D35A74501FCB}.Release|x86.ActiveCfg = Release|Any CPU - {8E7B180C-7057-4C81-BD0F-D35A74501FCB}.Release|x86.Build.0 = Release|Any CPU {89162E83-DEFA-4474-B6AE-669EB64374DB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {89162E83-DEFA-4474-B6AE-669EB64374DB}.Debug|Any CPU.Build.0 = Debug|Any CPU - {89162E83-DEFA-4474-B6AE-669EB64374DB}.Debug|x64.ActiveCfg = Debug|Any CPU - {89162E83-DEFA-4474-B6AE-669EB64374DB}.Debug|x64.Build.0 = Debug|Any CPU - {89162E83-DEFA-4474-B6AE-669EB64374DB}.Debug|x86.ActiveCfg = Debug|Any CPU - {89162E83-DEFA-4474-B6AE-669EB64374DB}.Debug|x86.Build.0 = Debug|Any CPU {89162E83-DEFA-4474-B6AE-669EB64374DB}.Release|Any CPU.ActiveCfg = Release|Any CPU {89162E83-DEFA-4474-B6AE-669EB64374DB}.Release|Any CPU.Build.0 = Release|Any CPU - {89162E83-DEFA-4474-B6AE-669EB64374DB}.Release|x64.ActiveCfg = Release|Any CPU - {89162E83-DEFA-4474-B6AE-669EB64374DB}.Release|x64.Build.0 = Release|Any CPU - {89162E83-DEFA-4474-B6AE-669EB64374DB}.Release|x86.ActiveCfg = Release|Any CPU - {89162E83-DEFA-4474-B6AE-669EB64374DB}.Release|x86.Build.0 = Release|Any CPU {C498BDCC-CD27-4867-85AA-00EFA5EE3809}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {C498BDCC-CD27-4867-85AA-00EFA5EE3809}.Debug|Any CPU.Build.0 = Debug|Any CPU - {C498BDCC-CD27-4867-85AA-00EFA5EE3809}.Debug|x64.ActiveCfg = Debug|Any CPU - {C498BDCC-CD27-4867-85AA-00EFA5EE3809}.Debug|x64.Build.0 = Debug|Any CPU - {C498BDCC-CD27-4867-85AA-00EFA5EE3809}.Debug|x86.ActiveCfg = Debug|Any CPU - {C498BDCC-CD27-4867-85AA-00EFA5EE3809}.Debug|x86.Build.0 = Debug|Any CPU {C498BDCC-CD27-4867-85AA-00EFA5EE3809}.Release|Any CPU.ActiveCfg = Release|Any CPU {C498BDCC-CD27-4867-85AA-00EFA5EE3809}.Release|Any CPU.Build.0 = Release|Any CPU - {C498BDCC-CD27-4867-85AA-00EFA5EE3809}.Release|x64.ActiveCfg = Release|Any CPU - {C498BDCC-CD27-4867-85AA-00EFA5EE3809}.Release|x64.Build.0 = Release|Any CPU - {C498BDCC-CD27-4867-85AA-00EFA5EE3809}.Release|x86.ActiveCfg = Release|Any CPU - {C498BDCC-CD27-4867-85AA-00EFA5EE3809}.Release|x86.Build.0 = Release|Any CPU {B86AF861-EB06-45A7-9132-E4DDB498C98C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {B86AF861-EB06-45A7-9132-E4DDB498C98C}.Debug|Any CPU.Build.0 = Debug|Any CPU - {B86AF861-EB06-45A7-9132-E4DDB498C98C}.Debug|x64.ActiveCfg = Debug|Any CPU - {B86AF861-EB06-45A7-9132-E4DDB498C98C}.Debug|x64.Build.0 = Debug|Any CPU - {B86AF861-EB06-45A7-9132-E4DDB498C98C}.Debug|x86.ActiveCfg = Debug|Any CPU - {B86AF861-EB06-45A7-9132-E4DDB498C98C}.Debug|x86.Build.0 = Debug|Any CPU {B86AF861-EB06-45A7-9132-E4DDB498C98C}.Release|Any CPU.ActiveCfg = Release|Any CPU {B86AF861-EB06-45A7-9132-E4DDB498C98C}.Release|Any CPU.Build.0 = Release|Any CPU - {B86AF861-EB06-45A7-9132-E4DDB498C98C}.Release|x64.ActiveCfg = Release|Any CPU - {B86AF861-EB06-45A7-9132-E4DDB498C98C}.Release|x64.Build.0 = Release|Any CPU - {B86AF861-EB06-45A7-9132-E4DDB498C98C}.Release|x86.ActiveCfg = Release|Any CPU - {B86AF861-EB06-45A7-9132-E4DDB498C98C}.Release|x86.Build.0 = Release|Any CPU {1B8B170B-4FD8-4CAE-97E3-4351DBD7939A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {1B8B170B-4FD8-4CAE-97E3-4351DBD7939A}.Debug|Any CPU.Build.0 = Debug|Any CPU - {1B8B170B-4FD8-4CAE-97E3-4351DBD7939A}.Debug|x64.ActiveCfg = Debug|Any CPU - {1B8B170B-4FD8-4CAE-97E3-4351DBD7939A}.Debug|x64.Build.0 = Debug|Any CPU - {1B8B170B-4FD8-4CAE-97E3-4351DBD7939A}.Debug|x86.ActiveCfg = Debug|Any CPU - {1B8B170B-4FD8-4CAE-97E3-4351DBD7939A}.Debug|x86.Build.0 = Debug|Any CPU {1B8B170B-4FD8-4CAE-97E3-4351DBD7939A}.Release|Any CPU.ActiveCfg = Release|Any CPU {1B8B170B-4FD8-4CAE-97E3-4351DBD7939A}.Release|Any CPU.Build.0 = Release|Any CPU - {1B8B170B-4FD8-4CAE-97E3-4351DBD7939A}.Release|x64.ActiveCfg = Release|Any CPU - {1B8B170B-4FD8-4CAE-97E3-4351DBD7939A}.Release|x64.Build.0 = Release|Any CPU - {1B8B170B-4FD8-4CAE-97E3-4351DBD7939A}.Release|x86.ActiveCfg = Release|Any CPU - {1B8B170B-4FD8-4CAE-97E3-4351DBD7939A}.Release|x86.Build.0 = Release|Any CPU {8ED5D3DC-86F7-4599-8ED5-C0E20FFDE7F6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {8ED5D3DC-86F7-4599-8ED5-C0E20FFDE7F6}.Debug|Any CPU.Build.0 = Debug|Any CPU - {8ED5D3DC-86F7-4599-8ED5-C0E20FFDE7F6}.Debug|x64.ActiveCfg = Debug|Any CPU - {8ED5D3DC-86F7-4599-8ED5-C0E20FFDE7F6}.Debug|x64.Build.0 = Debug|Any CPU - {8ED5D3DC-86F7-4599-8ED5-C0E20FFDE7F6}.Debug|x86.ActiveCfg = Debug|Any CPU - {8ED5D3DC-86F7-4599-8ED5-C0E20FFDE7F6}.Debug|x86.Build.0 = Debug|Any CPU {8ED5D3DC-86F7-4599-8ED5-C0E20FFDE7F6}.Release|Any CPU.ActiveCfg = Release|Any CPU {8ED5D3DC-86F7-4599-8ED5-C0E20FFDE7F6}.Release|Any CPU.Build.0 = Release|Any CPU - {8ED5D3DC-86F7-4599-8ED5-C0E20FFDE7F6}.Release|x64.ActiveCfg = Release|Any CPU - {8ED5D3DC-86F7-4599-8ED5-C0E20FFDE7F6}.Release|x64.Build.0 = Release|Any CPU - {8ED5D3DC-86F7-4599-8ED5-C0E20FFDE7F6}.Release|x86.ActiveCfg = Release|Any CPU - {8ED5D3DC-86F7-4599-8ED5-C0E20FFDE7F6}.Release|x86.Build.0 = Release|Any CPU {0527FB48-60E0-4D38-B3D9-0A889E1FFAB2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {0527FB48-60E0-4D38-B3D9-0A889E1FFAB2}.Debug|Any CPU.Build.0 = Debug|Any CPU - {0527FB48-60E0-4D38-B3D9-0A889E1FFAB2}.Debug|x64.ActiveCfg = Debug|Any CPU - {0527FB48-60E0-4D38-B3D9-0A889E1FFAB2}.Debug|x64.Build.0 = Debug|Any CPU - {0527FB48-60E0-4D38-B3D9-0A889E1FFAB2}.Debug|x86.ActiveCfg = Debug|Any CPU - {0527FB48-60E0-4D38-B3D9-0A889E1FFAB2}.Debug|x86.Build.0 = Debug|Any CPU {0527FB48-60E0-4D38-B3D9-0A889E1FFAB2}.Release|Any CPU.ActiveCfg = Release|Any CPU {0527FB48-60E0-4D38-B3D9-0A889E1FFAB2}.Release|Any CPU.Build.0 = Release|Any CPU - {0527FB48-60E0-4D38-B3D9-0A889E1FFAB2}.Release|x64.ActiveCfg = Release|Any CPU - {0527FB48-60E0-4D38-B3D9-0A889E1FFAB2}.Release|x64.Build.0 = Release|Any CPU - {0527FB48-60E0-4D38-B3D9-0A889E1FFAB2}.Release|x86.ActiveCfg = Release|Any CPU - {0527FB48-60E0-4D38-B3D9-0A889E1FFAB2}.Release|x86.Build.0 = Release|Any CPU {1EA44258-04FC-48D5-A60D-AE92214DEB1A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {1EA44258-04FC-48D5-A60D-AE92214DEB1A}.Debug|Any CPU.Build.0 = Debug|Any CPU - {1EA44258-04FC-48D5-A60D-AE92214DEB1A}.Debug|x64.ActiveCfg = Debug|Any CPU - {1EA44258-04FC-48D5-A60D-AE92214DEB1A}.Debug|x64.Build.0 = Debug|Any CPU - {1EA44258-04FC-48D5-A60D-AE92214DEB1A}.Debug|x86.ActiveCfg = Debug|Any CPU - {1EA44258-04FC-48D5-A60D-AE92214DEB1A}.Debug|x86.Build.0 = Debug|Any CPU {1EA44258-04FC-48D5-A60D-AE92214DEB1A}.Release|Any CPU.ActiveCfg = Release|Any CPU {1EA44258-04FC-48D5-A60D-AE92214DEB1A}.Release|Any CPU.Build.0 = Release|Any CPU - {1EA44258-04FC-48D5-A60D-AE92214DEB1A}.Release|x64.ActiveCfg = Release|Any CPU - {1EA44258-04FC-48D5-A60D-AE92214DEB1A}.Release|x64.Build.0 = Release|Any CPU - {1EA44258-04FC-48D5-A60D-AE92214DEB1A}.Release|x86.ActiveCfg = Release|Any CPU - {1EA44258-04FC-48D5-A60D-AE92214DEB1A}.Release|x86.Build.0 = Release|Any CPU {0B6CBF18-D11C-4CC3-A50C-437A9F7E761D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {0B6CBF18-D11C-4CC3-A50C-437A9F7E761D}.Debug|Any CPU.Build.0 = Debug|Any CPU - {0B6CBF18-D11C-4CC3-A50C-437A9F7E761D}.Debug|x64.ActiveCfg = Debug|Any CPU - {0B6CBF18-D11C-4CC3-A50C-437A9F7E761D}.Debug|x64.Build.0 = Debug|Any CPU - {0B6CBF18-D11C-4CC3-A50C-437A9F7E761D}.Debug|x86.ActiveCfg = Debug|Any CPU - {0B6CBF18-D11C-4CC3-A50C-437A9F7E761D}.Debug|x86.Build.0 = Debug|Any CPU {0B6CBF18-D11C-4CC3-A50C-437A9F7E761D}.Release|Any CPU.ActiveCfg = Release|Any CPU {0B6CBF18-D11C-4CC3-A50C-437A9F7E761D}.Release|Any CPU.Build.0 = Release|Any CPU - {0B6CBF18-D11C-4CC3-A50C-437A9F7E761D}.Release|x64.ActiveCfg = Release|Any CPU - {0B6CBF18-D11C-4CC3-A50C-437A9F7E761D}.Release|x64.Build.0 = Release|Any CPU - {0B6CBF18-D11C-4CC3-A50C-437A9F7E761D}.Release|x86.ActiveCfg = Release|Any CPU - {0B6CBF18-D11C-4CC3-A50C-437A9F7E761D}.Release|x86.Build.0 = Release|Any CPU {1F6C87C7-1911-4E96-ADB4-1EF27BD09C60}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {1F6C87C7-1911-4E96-ADB4-1EF27BD09C60}.Debug|Any CPU.Build.0 = Debug|Any CPU - {1F6C87C7-1911-4E96-ADB4-1EF27BD09C60}.Debug|x64.ActiveCfg = Debug|Any CPU - {1F6C87C7-1911-4E96-ADB4-1EF27BD09C60}.Debug|x64.Build.0 = Debug|Any CPU - {1F6C87C7-1911-4E96-ADB4-1EF27BD09C60}.Debug|x86.ActiveCfg = Debug|Any CPU - {1F6C87C7-1911-4E96-ADB4-1EF27BD09C60}.Debug|x86.Build.0 = Debug|Any CPU {1F6C87C7-1911-4E96-ADB4-1EF27BD09C60}.Release|Any CPU.ActiveCfg = Release|Any CPU {1F6C87C7-1911-4E96-ADB4-1EF27BD09C60}.Release|Any CPU.Build.0 = Release|Any CPU - {1F6C87C7-1911-4E96-ADB4-1EF27BD09C60}.Release|x64.ActiveCfg = Release|Any CPU - {1F6C87C7-1911-4E96-ADB4-1EF27BD09C60}.Release|x64.Build.0 = Release|Any CPU - {1F6C87C7-1911-4E96-ADB4-1EF27BD09C60}.Release|x86.ActiveCfg = Release|Any CPU - {1F6C87C7-1911-4E96-ADB4-1EF27BD09C60}.Release|x86.Build.0 = Release|Any CPU {71C2BC96-C104-4748-B6CF-E74BAC4E709D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {71C2BC96-C104-4748-B6CF-E74BAC4E709D}.Debug|Any CPU.Build.0 = Debug|Any CPU - {71C2BC96-C104-4748-B6CF-E74BAC4E709D}.Debug|x64.ActiveCfg = Debug|Any CPU - {71C2BC96-C104-4748-B6CF-E74BAC4E709D}.Debug|x64.Build.0 = Debug|Any CPU - {71C2BC96-C104-4748-B6CF-E74BAC4E709D}.Debug|x86.ActiveCfg = Debug|Any CPU - {71C2BC96-C104-4748-B6CF-E74BAC4E709D}.Debug|x86.Build.0 = Debug|Any CPU {71C2BC96-C104-4748-B6CF-E74BAC4E709D}.Release|Any CPU.ActiveCfg = Release|Any CPU {71C2BC96-C104-4748-B6CF-E74BAC4E709D}.Release|Any CPU.Build.0 = Release|Any CPU - {71C2BC96-C104-4748-B6CF-E74BAC4E709D}.Release|x64.ActiveCfg = Release|Any CPU - {71C2BC96-C104-4748-B6CF-E74BAC4E709D}.Release|x64.Build.0 = Release|Any CPU - {71C2BC96-C104-4748-B6CF-E74BAC4E709D}.Release|x86.ActiveCfg = Release|Any CPU - {71C2BC96-C104-4748-B6CF-E74BAC4E709D}.Release|x86.Build.0 = Release|Any CPU {21649885-B40A-4F6B-9286-A7E378276C3E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {21649885-B40A-4F6B-9286-A7E378276C3E}.Debug|Any CPU.Build.0 = Debug|Any CPU - {21649885-B40A-4F6B-9286-A7E378276C3E}.Debug|x64.ActiveCfg = Debug|Any CPU - {21649885-B40A-4F6B-9286-A7E378276C3E}.Debug|x64.Build.0 = Debug|Any CPU - {21649885-B40A-4F6B-9286-A7E378276C3E}.Debug|x86.ActiveCfg = Debug|Any CPU - {21649885-B40A-4F6B-9286-A7E378276C3E}.Debug|x86.Build.0 = Debug|Any CPU {21649885-B40A-4F6B-9286-A7E378276C3E}.Release|Any CPU.ActiveCfg = Release|Any CPU {21649885-B40A-4F6B-9286-A7E378276C3E}.Release|Any CPU.Build.0 = Release|Any CPU - {21649885-B40A-4F6B-9286-A7E378276C3E}.Release|x64.ActiveCfg = Release|Any CPU - {21649885-B40A-4F6B-9286-A7E378276C3E}.Release|x64.Build.0 = Release|Any CPU - {21649885-B40A-4F6B-9286-A7E378276C3E}.Release|x86.ActiveCfg = Release|Any CPU - {21649885-B40A-4F6B-9286-A7E378276C3E}.Release|x86.Build.0 = Release|Any CPU {DA8E9EC2-990C-4DA2-A3F3-7E7796C71DD8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {DA8E9EC2-990C-4DA2-A3F3-7E7796C71DD8}.Debug|Any CPU.Build.0 = Debug|Any CPU - {DA8E9EC2-990C-4DA2-A3F3-7E7796C71DD8}.Debug|x64.ActiveCfg = Debug|Any CPU - {DA8E9EC2-990C-4DA2-A3F3-7E7796C71DD8}.Debug|x64.Build.0 = Debug|Any CPU - {DA8E9EC2-990C-4DA2-A3F3-7E7796C71DD8}.Debug|x86.ActiveCfg = Debug|Any CPU - {DA8E9EC2-990C-4DA2-A3F3-7E7796C71DD8}.Debug|x86.Build.0 = Debug|Any CPU {DA8E9EC2-990C-4DA2-A3F3-7E7796C71DD8}.Release|Any CPU.ActiveCfg = Release|Any CPU {DA8E9EC2-990C-4DA2-A3F3-7E7796C71DD8}.Release|Any CPU.Build.0 = Release|Any CPU - {DA8E9EC2-990C-4DA2-A3F3-7E7796C71DD8}.Release|x64.ActiveCfg = Release|Any CPU - {DA8E9EC2-990C-4DA2-A3F3-7E7796C71DD8}.Release|x64.Build.0 = Release|Any CPU - {DA8E9EC2-990C-4DA2-A3F3-7E7796C71DD8}.Release|x86.ActiveCfg = Release|Any CPU - {DA8E9EC2-990C-4DA2-A3F3-7E7796C71DD8}.Release|x86.Build.0 = Release|Any CPU - {B54702BF-ECAD-4F96-BC43-F5B670F0A668}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {B54702BF-ECAD-4F96-BC43-F5B670F0A668}.Debug|Any CPU.Build.0 = Debug|Any CPU - {B54702BF-ECAD-4F96-BC43-F5B670F0A668}.Debug|x64.ActiveCfg = Debug|Any CPU - {B54702BF-ECAD-4F96-BC43-F5B670F0A668}.Debug|x64.Build.0 = Debug|Any CPU - {B54702BF-ECAD-4F96-BC43-F5B670F0A668}.Debug|x86.ActiveCfg = Debug|Any CPU - {B54702BF-ECAD-4F96-BC43-F5B670F0A668}.Debug|x86.Build.0 = Debug|Any CPU - {B54702BF-ECAD-4F96-BC43-F5B670F0A668}.Release|Any CPU.ActiveCfg = Release|Any CPU - {B54702BF-ECAD-4F96-BC43-F5B670F0A668}.Release|Any CPU.Build.0 = Release|Any CPU - {B54702BF-ECAD-4F96-BC43-F5B670F0A668}.Release|x64.ActiveCfg = Release|Any CPU - {B54702BF-ECAD-4F96-BC43-F5B670F0A668}.Release|x64.Build.0 = Release|Any CPU - {B54702BF-ECAD-4F96-BC43-F5B670F0A668}.Release|x86.ActiveCfg = Release|Any CPU - {B54702BF-ECAD-4F96-BC43-F5B670F0A668}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -866,7 +432,6 @@ Global {71C2BC96-C104-4748-B6CF-E74BAC4E709D} = {4827B3EC-73D8-436D-AE2A-5E29AC95FD0C} {21649885-B40A-4F6B-9286-A7E378276C3E} = {C526E8AB-739A-48D7-8FC4-048978C9B650} {DA8E9EC2-990C-4DA2-A3F3-7E7796C71DD8} = {C526E8AB-739A-48D7-8FC4-048978C9B650} - {B54702BF-ECAD-4F96-BC43-F5B670F0A668} = {95A69671-16CA-4133-981C-CC381B7AAA30} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {8CBB7278-D093-448E-B3DE-B5991209A1AA} diff --git a/sandbox/SubCleanDispose/Program.cs b/sandbox/SubCleanDispose/Program.cs deleted file mode 100644 index bce47ffa3..000000000 --- a/sandbox/SubCleanDispose/Program.cs +++ /dev/null @@ -1,227 +0,0 @@ -// Manual dispose-race repro for NATS subscriptions and JetStream consumers. -// -// Each scenario publishes N messages with sequential payloads, starts a -// subscriber/consumer, cancels mid-flight, then checks the received set for -// gaps. A gap (received {1, 2, 4} with no 3) means a message was dropped -// during dispose. -// -// Usage: SubCleanDispose [iterations] -// NATS_URL env var overrides the server URL (default nats://127.0.0.1:4222). -using System.Diagnostics; -using NATS.Client.Core; -using NATS.Client.JetStream; -using NATS.Client.JetStream.Models; - -var url = Environment.GetEnvironmentVariable("NATS_URL") ?? "nats://127.0.0.1:4222"; -var iterations = args.Length > 0 && int.TryParse(args[0], out var n) ? n : 20; - -Console.WriteLine($"url={url} iterations={iterations}"); -Console.WriteLine(); - -var failures = new Dictionary -{ - ["core-sub"] = 0, - ["js-fetch"] = 0, - ["js-consume"] = 0, - ["js-ordered"] = 0, -}; - -for (var i = 1; i <= iterations; i++) -{ - await RunOnce("core-sub", i, () => TestCoreSub(url)); - await RunOnce("js-fetch", i, () => TestJsFetch(url)); - await RunOnce("js-consume", i, () => TestJsConsume(url)); - await RunOnce("js-ordered", i, () => TestJsOrdered(url)); -} - -Console.WriteLine(); -Console.WriteLine("Summary:"); -foreach (var (name, count) in failures) - Console.WriteLine($" {name,-12} {count} failure(s)"); - -return failures.Values.Sum() == 0 ? 0 : 1; - -async Task RunOnce(string name, int iter, Func> body) -{ - var sw = Stopwatch.StartNew(); - Result r; - try - { - r = await body(); - } - catch (Exception e) - { - Console.WriteLine($"[{name}] iter {iter} ERROR {e.GetType().Name}: {e.Message}"); - failures[name]++; - return; - } - - sw.Stop(); - var ok = r.Gap == 0; - if (!ok) - failures[name]++; - Console.WriteLine( - $"[{name}] iter {iter,3} published={r.Published,5} received={r.Received,5} highest={r.Highest,5} gap={r.Gap,4} {sw.ElapsedMilliseconds,5}ms {(ok ? "ok" : "DROP")}"); -} - -static async Task TestCoreSub(string url) -{ - const int published = 1000; - await using var nats = new NatsConnection(new NatsOpts { Url = url }); - await nats.ConnectAsync(); - - var subject = $"t.{Guid.NewGuid():N}"; - var received = new HashSet(); - - await using var sub = await nats.SubscribeCoreAsync(subject); - await nats.PingAsync(); - - var readerDone = new TaskCompletionSource(); - var reader = Task.Run(async () => - { - try - { - await foreach (var msg in sub.Msgs.ReadAllAsync()) - { - received.Add(msg.Data); - if (received.Count == 10) - readerDone.TrySetResult(); - } - } - catch (Exception) - { - } - }); - - for (var i = 1; i <= published; i++) - await nats.PublishAsync(subject, i); - - await readerDone.Task; - - // Dispose while messages may still be in-flight. Core NATS has no - // server-side ack tracking, so gaps here indicate drop-on-dispose. - await sub.DisposeAsync(); - await reader; - - return Result.From(published, received); -} - -static async Task TestJsFetch(string url) -{ - const int published = 1000; - await using var nats = new NatsConnection(new NatsOpts { Url = url }); - var js = new NatsJSContext(nats); - var id = Guid.NewGuid().ToString("N").Substring(0, 8); - var stream = $"s_{id}"; - var subject = $"s_{id}.x"; - var consumer = $"c_{id}"; - - await js.CreateStreamAsync(new StreamConfig(stream, new[] { subject })); - var c = await js.CreateOrUpdateConsumerAsync(stream, new ConsumerConfig(consumer) { AckPolicy = ConsumerConfigAckPolicy.Explicit }); - - for (var i = 1; i <= published; i++) - await js.PublishAsync(subject, i); - - var received = new HashSet(); - using var cts = new CancellationTokenSource(); - try - { - await foreach (var m in c.FetchAsync(new NatsJSFetchOpts { MaxMsgs = published, Expires = TimeSpan.FromSeconds(10) }, cancellationToken: cts.Token)) - { - received.Add(m.Data); - await m.AckAsync(); - if (received.Count == 10) - cts.CancelAfter(TimeSpan.FromMilliseconds(1)); - } - } - catch (OperationCanceledException) - { - } - - await js.DeleteStreamAsync(stream); - return Result.From(published, received); -} - -static async Task TestJsConsume(string url) -{ - const int published = 1000; - await using var nats = new NatsConnection(new NatsOpts { Url = url }); - var js = new NatsJSContext(nats); - var id = Guid.NewGuid().ToString("N").Substring(0, 8); - var stream = $"s_{id}"; - var subject = $"s_{id}.x"; - var consumer = $"c_{id}"; - - await js.CreateStreamAsync(new StreamConfig(stream, new[] { subject })); - var c = await js.CreateOrUpdateConsumerAsync(stream, new ConsumerConfig(consumer) { AckPolicy = ConsumerConfigAckPolicy.Explicit }); - - for (var i = 1; i <= published; i++) - await js.PublishAsync(subject, i); - - var received = new HashSet(); - using var cts = new CancellationTokenSource(); - try - { - await foreach (var m in c.ConsumeAsync(opts: new NatsJSConsumeOpts { MaxMsgs = published }, cancellationToken: cts.Token)) - { - received.Add(m.Data); - await m.AckAsync(); - if (received.Count == 10) - cts.CancelAfter(TimeSpan.FromMilliseconds(1)); - } - } - catch (OperationCanceledException) - { - } - - await js.DeleteStreamAsync(stream); - return Result.From(published, received); -} - -static async Task TestJsOrdered(string url) -{ - const int published = 1000; - await using var nats = new NatsConnection(new NatsOpts { Url = url }); - var js = new NatsJSContext(nats); - var id = Guid.NewGuid().ToString("N").Substring(0, 8); - var stream = $"s_{id}"; - var subject = $"s_{id}.x"; - - await js.CreateStreamAsync(new StreamConfig(stream, new[] { subject })); - var c = await js.CreateOrderedConsumerAsync(stream, new NatsJSOrderedConsumerOpts { FilterSubjects = new[] { subject } }); - - for (var i = 1; i <= published; i++) - await js.PublishAsync(subject, i); - - var received = new HashSet(); - using var cts = new CancellationTokenSource(); - try - { - await foreach (var m in c.ConsumeAsync(cancellationToken: cts.Token)) - { - received.Add(m.Data); - if (received.Count == 10) - cts.CancelAfter(TimeSpan.FromMilliseconds(1)); - } - } - catch (OperationCanceledException) - { - } - - await js.DeleteStreamAsync(stream); - return Result.From(published, received); -} - -internal readonly record struct Result(int Published, int Received, int Highest, int Gap) -{ - public static Result From(int published, HashSet received) - { - if (received.Count == 0) - return new Result(published, 0, 0, 0); - var max = received.Max(); - - // Gap = "how many integers in [1..max] are missing from received". - var gap = max - received.Count; - return new Result(published, received.Count, max, gap); - } -} diff --git a/sandbox/SubCleanDispose/SubCleanDispose.csproj b/sandbox/SubCleanDispose/SubCleanDispose.csproj deleted file mode 100644 index befc5a0df..000000000 --- a/sandbox/SubCleanDispose/SubCleanDispose.csproj +++ /dev/null @@ -1,16 +0,0 @@ - - - - Exe - net8.0 - enable - enable - false - - - - - - - - diff --git a/src/NATS.Client.Core/NatsConnection.cs b/src/NATS.Client.Core/NatsConnection.cs index b814a3676..7fee532e7 100644 --- a/src/NATS.Client.Core/NatsConnection.cs +++ b/src/NATS.Client.Core/NatsConnection.cs @@ -280,7 +280,6 @@ public virtual async ValueTask DisposeAsync() IsDisposed = true; _logger.Log(LogLevel.Information, NatsLogEvents.Connection, "Disposing connection {Name}", _name); - await DisposeSocketAsync(false).ConfigureAwait(false); if (_pingTimerCancellationTokenSource != null) { #if NET8_0_OR_GREATER @@ -290,8 +289,11 @@ public virtual async ValueTask DisposeAsync() #endif } + // Drain subscriptions and flush the writer before tearing down + // the socket so UNSUB/PING/PONG and pending acks can land. await _subscriptionManager.DisposeAsync().ConfigureAwait(false); await CommandWriter.DisposeAsync().ConfigureAwait(false); + await DisposeSocketAsync(false).ConfigureAwait(false); _waitForOpenConnection.TrySetCanceled(); #if NET8_0_OR_GREATER await _disposedCts.CancelAsync().ConfigureAwait(false); diff --git a/src/NATS.Client.Core/NatsSubBase.cs b/src/NATS.Client.Core/NatsSubBase.cs index 23fdc6387..af69cce2d 100644 --- a/src/NATS.Client.Core/NatsSubBase.cs +++ b/src/NATS.Client.Core/NatsSubBase.cs @@ -404,6 +404,11 @@ internal async ValueTask DrainAsync() { _logger.LogWarning(NatsLogEvents.Subscription, e, "Error while completing subscription"); } + + // Hook for subclasses to wait for an active user reader (e.g. an + // await-foreach loop) to drain the channel before the connection + // tears down the writer/socket. + await WaitForReaderDrainAsync().ConfigureAwait(false); } /// @@ -438,6 +443,12 @@ internal async ValueTask DrainAsync() /// protected abstract ValueTask ReceiveInternalAsync(string subject, string? replyTo, ReadOnlySequence? headersBuffer, ReadOnlySequence payloadBuffer); + /// + /// Subclasses override to wait for the user-side reader to finish + /// draining buffered messages. Default is no-op. + /// + protected virtual ValueTask WaitForReaderDrainAsync() => default; + /// /// Resets the slow consumer state if the channel has drained, allowing another /// slow consumer event to be raised if the subscription becomes slow again. diff --git a/src/NATS.Client.JetStream/Internal/NatsJSConsume.cs b/src/NATS.Client.JetStream/Internal/NatsJSConsume.cs index daefaaa14..5821bd123 100644 --- a/src/NATS.Client.JetStream/Internal/NatsJSConsume.cs +++ b/src/NATS.Client.JetStream/Internal/NatsJSConsume.cs @@ -43,10 +43,12 @@ internal class NatsJSConsume : NatsSubBase private readonly int _maxConsecutive503Errors; private readonly object _pendingGate = new(); + private readonly TaskCompletionSource _readerExited = new(TaskCreationOptions.RunContinuationsAsynchronously); private long _pendingMsgs; private long _pendingBytes; private int _disposed; private int _consecutive503Errors; + private int _readerActive; public NatsJSConsume( long maxMsgs, @@ -197,6 +199,14 @@ public ValueTask CallMsgNextAsync(string origin, ConsumerGetnextRequest request, public void ResetHeartbeatTimer() => _timer.Change(_hbTimeout, _hbTimeout); + public void MarkReaderActive() => Interlocked.Exchange(ref _readerActive, 1); + + public void MarkReaderInactive() + { + Interlocked.Exchange(ref _readerActive, 0); + _readerExited.TrySetResult(); + } + public void Delivered(int msgSize) { lock (_pendingGate) @@ -299,6 +309,22 @@ await commandWriter.PublishAsync( } } + protected override async ValueTask WaitForReaderDrainAsync() + { + if (Volatile.Read(ref _readerActive) == 0) + return; + + try + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + await _readerExited.Task.WaitAsync(cts.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + _logger.LogWarning(NatsJSLogEvents.Internal, "Timeout waiting for consumer reader to exit on dispose"); + } + } + protected override async ValueTask ReceiveInternalAsync( string subject, string? replyTo, diff --git a/src/NATS.Client.JetStream/Internal/NatsJSFetch.cs b/src/NATS.Client.JetStream/Internal/NatsJSFetch.cs index 015d0bb58..777489ae5 100644 --- a/src/NATS.Client.JetStream/Internal/NatsJSFetch.cs +++ b/src/NATS.Client.JetStream/Internal/NatsJSFetch.cs @@ -30,9 +30,11 @@ internal class NatsJSFetch : NatsSubBase private readonly TimeSpan _idle; private readonly long _hbTimeout; + private readonly TaskCompletionSource _readerExited = new(TaskCreationOptions.RunContinuationsAsynchronously); private long _pendingMsgs; private long _pendingBytes; private int _disposed; + private int _readerActive; public NatsJSFetch( long maxMsgs, @@ -160,6 +162,14 @@ public void ResetHeartbeatTimer() _hbTimer.Change(_hbTimeout, Timeout.Infinite); } + public void MarkReaderActive() => Interlocked.Exchange(ref _readerActive, 1); + + public void MarkReaderInactive() + { + Interlocked.Exchange(ref _readerActive, 0); + _readerExited.TrySetResult(); + } + public override async ValueTask DisposeAsync() { Interlocked.Exchange(ref _disposed, 1); @@ -210,6 +220,22 @@ await commandWriter.PublishAsync( cancellationToken: CancellationToken.None); } + protected override async ValueTask WaitForReaderDrainAsync() + { + if (Volatile.Read(ref _readerActive) == 0) + return; + + try + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + await _readerExited.Task.WaitAsync(cts.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + _logger.LogWarning(NatsJSLogEvents.Internal, "Timeout waiting for fetch reader to exit on dispose"); + } + } + protected override async ValueTask ReceiveInternalAsync( string subject, string? replyTo, diff --git a/src/NATS.Client.JetStream/Internal/NatsJSOrderedConsume.cs b/src/NATS.Client.JetStream/Internal/NatsJSOrderedConsume.cs index 09cada970..b23950650 100644 --- a/src/NATS.Client.JetStream/Internal/NatsJSOrderedConsume.cs +++ b/src/NATS.Client.JetStream/Internal/NatsJSOrderedConsume.cs @@ -32,9 +32,11 @@ internal class NatsJSOrderedConsume : NatsSubBase private readonly long _thresholdBytes; private readonly object _pendingGate = new(); + private readonly TaskCompletionSource _readerExited = new(TaskCreationOptions.RunContinuationsAsynchronously); private long _pendingMsgs; private long _pendingBytes; private int _disposed; + private int _readerActive; public NatsJSOrderedConsume( long maxMsgs, @@ -132,6 +134,14 @@ public ValueTask CallMsgNextAsync(string origin, ConsumerGetnextRequest request, public void ResetHeartbeatTimer() => _timer.Change(_hbTimeout, Timeout.Infinite); + public void MarkReaderActive() => Interlocked.Exchange(ref _readerActive, 1); + + public void MarkReaderInactive() + { + Interlocked.Exchange(ref _readerActive, 0); + _readerExited.TrySetResult(); + } + public override async ValueTask DisposeAsync() { Interlocked.Exchange(ref _disposed, 1); @@ -161,6 +171,22 @@ internal override ValueTask WriteReconnectCommandsAsync(CommandWriter commandWri return default; } + protected override async ValueTask WaitForReaderDrainAsync() + { + if (Volatile.Read(ref _readerActive) == 0) + return; + + try + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + await _readerExited.Task.WaitAsync(cts.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + _logger.LogWarning(NatsJSLogEvents.Internal, "Timeout waiting for ordered-consume reader to exit on dispose"); + } + } + protected override async ValueTask ReceiveInternalAsync( string subject, string? replyTo, diff --git a/src/NATS.Client.JetStream/NatsJSConsumer.cs b/src/NATS.Client.JetStream/NatsJSConsumer.cs index 996bfacf2..e10958563 100644 --- a/src/NATS.Client.JetStream/NatsJSConsumer.cs +++ b/src/NATS.Client.JetStream/NatsJSConsumer.cs @@ -63,45 +63,20 @@ public async IAsyncEnumerable> ConsumeAsync( { opts ??= _context.Opts.DefaultConsumeOpts; await using var cc = await ConsumeInternalAsync(serializer, opts, cancellationToken).ConfigureAwait(false); - - while (!cancellationToken.IsCancellationRequested) + cc.MarkReaderActive(); + try { - // We have to check calls individually since we can't use yield return in try-catch blocks. - bool ready; - try - { - ready = await cc.Msgs.WaitToReadAsync(cancellationToken).ConfigureAwait(false); - } - catch (OperationCanceledException) - { - ready = false; - } - catch (NatsConnectionFailedException) - { - // Connection has permanently failed, stop consuming and rethrow - throw; - } - catch (NatsJSException) - { - // Consumer-related errors (like 503 threshold exceeded), stop consuming and rethrow - throw; - } - - if (!ready) - yield break; - while (!cancellationToken.IsCancellationRequested) { - bool read; - NatsJSMsg jsMsg; + // We have to check calls individually since we can't use yield return in try-catch blocks. + bool ready; try { - read = cc.Msgs.TryRead(out jsMsg); + ready = await cc.Msgs.WaitToReadAsync(cancellationToken).ConfigureAwait(false); } catch (OperationCanceledException) { - read = false; - jsMsg = default; + ready = false; } catch (NatsConnectionFailedException) { @@ -114,17 +89,49 @@ public async IAsyncEnumerable> ConsumeAsync( throw; } - if (!read) - break; + if (!ready) + yield break; - // if yield is blocked by the application code, we don't want - // heartbeat timer kicking in and issuing unnecessary pull requests. - cc.StopHeartbeatTimer(); - yield return jsMsg; - cc.ResetHeartbeatTimer(); - cc.Delivered(jsMsg.Size); + while (!cancellationToken.IsCancellationRequested) + { + bool read; + NatsJSMsg jsMsg; + try + { + read = cc.Msgs.TryRead(out jsMsg); + } + catch (OperationCanceledException) + { + read = false; + jsMsg = default; + } + catch (NatsConnectionFailedException) + { + // Connection has permanently failed, stop consuming and rethrow + throw; + } + catch (NatsJSException) + { + // Consumer-related errors (like 503 threshold exceeded), stop consuming and rethrow + throw; + } + + if (!read) + break; + + // if yield is blocked by the application code, we don't want + // heartbeat timer kicking in and issuing unnecessary pull requests. + cc.StopHeartbeatTimer(); + yield return jsMsg; + cc.ResetHeartbeatTimer(); + cc.Delivered(jsMsg.Size); + } } } + finally + { + cc.MarkReaderInactive(); + } } /// @@ -177,9 +184,17 @@ public async IAsyncEnumerable> ConsumeAsync( serializer, cancellationToken: cancellationToken).ConfigureAwait(false); - await foreach (var natsJSMsg in f.Msgs.ReadAllAsync(cancellationToken).ConfigureAwait(false)) + f.MarkReaderActive(); + try { - return natsJSMsg; + await foreach (var natsJSMsg in f.Msgs.ReadAllAsync(cancellationToken).ConfigureAwait(false)) + { + return natsJSMsg; + } + } + finally + { + f.MarkReaderInactive(); } return null; @@ -196,47 +211,54 @@ public async IAsyncEnumerable> FetchAsync( serializer ??= _context.Connection.Opts.SerializerRegistry.GetDeserializer(); await using var fc = await FetchInternalAsync(opts, serializer, cancellationToken).ConfigureAwait(false); - - while (!cancellationToken.IsCancellationRequested) + fc.MarkReaderActive(); + try { - // We have to check calls individually since we can't use yield return in try-catch blocks. - bool ready; - try - { - ready = await fc.Msgs.WaitToReadAsync(cancellationToken).ConfigureAwait(false); - } - catch (OperationCanceledException) - { - ready = false; - } - - if (!ready) - yield break; - while (!cancellationToken.IsCancellationRequested) { - bool read; - NatsJSMsg jsMsg; + // We have to check calls individually since we can't use yield return in try-catch blocks. + bool ready; try { - read = fc.Msgs.TryRead(out jsMsg); + ready = await fc.Msgs.WaitToReadAsync(cancellationToken).ConfigureAwait(false); } catch (OperationCanceledException) { - read = false; - jsMsg = default; + ready = false; } - if (!read) - break; + if (!ready) + yield break; - // if yield is blocked by the application code, we don't want - // heartbeat timer kicking in and issuing unnecessary pull requests. - fc.StopHeartbeatTimer(); - yield return jsMsg; - fc.ResetHeartbeatTimer(); + while (!cancellationToken.IsCancellationRequested) + { + bool read; + NatsJSMsg jsMsg; + try + { + read = fc.Msgs.TryRead(out jsMsg); + } + catch (OperationCanceledException) + { + read = false; + jsMsg = default; + } + + if (!read) + break; + + // if yield is blocked by the application code, we don't want + // heartbeat timer kicking in and issuing unnecessary pull requests. + fc.StopHeartbeatTimer(); + yield return jsMsg; + fc.ResetHeartbeatTimer(); + } } } + finally + { + fc.MarkReaderInactive(); + } } /// @@ -294,13 +316,21 @@ public async IAsyncEnumerable> FetchNoWaitAsync( serializer ??= _context.Connection.Opts.SerializerRegistry.GetDeserializer(); await using var fc = await FetchInternalAsync(opts with { NoWait = true }, serializer, cancellationToken).ConfigureAwait(false); - await foreach (var jsMsg in fc.Msgs.ReadAllAsync(cancellationToken).ConfigureAwait(false)) + fc.MarkReaderActive(); + try + { + await foreach (var jsMsg in fc.Msgs.ReadAllAsync(cancellationToken).ConfigureAwait(false)) + { + // if yield is blocked by the application code, we don't want + // heartbeat timer kicking in and issuing unnecessary pull requests. + fc.StopHeartbeatTimer(); + yield return jsMsg; + fc.ResetHeartbeatTimer(); + } + } + finally { - // if yield is blocked by the application code, we don't want - // heartbeat timer kicking in and issuing unnecessary pull requests. - fc.StopHeartbeatTimer(); - yield return jsMsg; - fc.ResetHeartbeatTimer(); + fc.MarkReaderInactive(); } } diff --git a/src/NATS.Client.JetStream/NatsJSOrderedConsumer.cs b/src/NATS.Client.JetStream/NatsJSOrderedConsumer.cs index e8d51e3d5..023d04286 100644 --- a/src/NATS.Client.JetStream/NatsJSOrderedConsumer.cs +++ b/src/NATS.Client.JetStream/NatsJSOrderedConsumer.cs @@ -85,72 +85,80 @@ public async IAsyncEnumerable> ConsumeAsync( await using (var cc = await consumer.OrderedConsumeInternalAsync(serializer, opts, cancellationToken)) { - while (true) + cc.MarkReaderActive(); + try { - // We have to check every call to WaitToReadAsync and TryRead for - // protocol exceptions individually because we can't yield return - // within try-catch. - try - { - var read = await cc.Msgs.WaitToReadAsync(cancellationToken).ConfigureAwait(false); - if (!read) - break; - } - catch (OperationCanceledException) - { - break; - } - catch (NatsJSProtocolException pe) - { - protocolException = pe; - goto CONSUME_LOOP; - } - catch (NatsConnectionFailedException) - { - // Connection has permanently failed, stop consuming and rethrow - throw; - } - catch (NatsJSException e) when (e is not NatsJSProtocolException and not NatsJSConnectionException and not NatsJSTimeoutException) - { - // Consumer-related errors (like 503 threshold exceeded), stop consuming and rethrow - throw; - } - catch (NatsJSConnectionException e) - { - _logger.LogWarning(NatsJSLogEvents.Retry, "{Error}. Retrying...", e.Message); - goto CONSUME_LOOP; - } - catch (NatsJSTimeoutException e) - { - notificationHandler?.Invoke(NatsJSTimeoutNotification.Default, cancellationToken); - _logger.LogWarning(NatsJSLogEvents.Retry, "{Error}. Retrying...", e.Message); - goto CONSUME_LOOP; - } - while (true) { - NatsJSMsg msg; - - var canRead = cc.Msgs.TryRead(out msg); - if (!canRead) + // We have to check every call to WaitToReadAsync and TryRead for + // protocol exceptions individually because we can't yield return + // within try-catch. + try + { + var read = await cc.Msgs.WaitToReadAsync(cancellationToken).ConfigureAwait(false); + if (!read) + break; + } + catch (OperationCanceledException) + { break; - - if (msg.Metadata is not { } metadata) - continue; - - var expected = cseq + 1; - if (metadata.Sequence.Consumer != expected) + } + catch (NatsJSProtocolException pe) { - _logger.LogWarning(NatsJSLogEvents.Retry, "Consumer sequence mismatch. Expected {Expected}, was {SequenceConsumer}. Retrying...", expected, metadata.Sequence.Consumer); + protocolException = pe; goto CONSUME_LOOP; } + catch (NatsConnectionFailedException) + { + // Connection has permanently failed, stop consuming and rethrow + throw; + } + catch (NatsJSException e) when (e is not NatsJSProtocolException and not NatsJSConnectionException and not NatsJSTimeoutException) + { + // Consumer-related errors (like 503 threshold exceeded), stop consuming and rethrow + throw; + } + catch (NatsJSConnectionException e) + { + _logger.LogWarning(NatsJSLogEvents.Retry, "{Error}. Retrying...", e.Message); + goto CONSUME_LOOP; + } + catch (NatsJSTimeoutException e) + { + notificationHandler?.Invoke(NatsJSTimeoutNotification.Default, cancellationToken); + _logger.LogWarning(NatsJSLogEvents.Retry, "{Error}. Retrying...", e.Message); + goto CONSUME_LOOP; + } + + while (true) + { + NatsJSMsg msg; + + var canRead = cc.Msgs.TryRead(out msg); + if (!canRead) + break; - seq = metadata.Sequence.Stream; - cseq = metadata.Sequence.Consumer; + if (msg.Metadata is not { } metadata) + continue; - yield return msg; + var expected = cseq + 1; + if (metadata.Sequence.Consumer != expected) + { + _logger.LogWarning(NatsJSLogEvents.Retry, "Consumer sequence mismatch. Expected {Expected}, was {SequenceConsumer}. Retrying...", expected, metadata.Sequence.Consumer); + goto CONSUME_LOOP; + } + + seq = metadata.Sequence.Stream; + cseq = metadata.Sequence.Consumer; + + yield return msg; + } } } + finally + { + cc.MarkReaderInactive(); + } } CONSUME_LOOP: diff --git a/tests/NATS.Client.JetStream.Tests/ConsumerConsumeTest.cs b/tests/NATS.Client.JetStream.Tests/ConsumerConsumeTest.cs index 13ee99293..785594a0d 100644 --- a/tests/NATS.Client.JetStream.Tests/ConsumerConsumeTest.cs +++ b/tests/NATS.Client.JetStream.Tests/ConsumerConsumeTest.cs @@ -357,6 +357,79 @@ await Retry.Until( Assert.Equal(0, consumer.Info.NumAckPending); } + [Fact] + public async Task Consume_connection_dispose_drains_buffered_messages() + { + // Disposing the connection while a consume loop is mid-flight must + // drain the buffered messages and let the loop ack them, not race + // CommandWriter teardown and throw ObjectDisposedException. + const int totalMsgs = 100; + const int pullBatch = 20; + const int bailAt = 10; + + var prefix = _server.GetNextId(); + var streamName = $"{prefix}s1"; + var subject = $"{prefix}s1.x"; + var consumerName = $"{prefix}c1"; + + var cts = new CancellationTokenSource(TimeSpan.FromSeconds(60)); + + await using (var setup = new NatsConnection(new NatsOpts { Url = _server.Url })) + { + var setupJs = new NatsJSContext(setup); + var stream = await setupJs.CreateStreamAsync(new StreamConfig(streamName, [subject]), cts.Token); + + for (var i = 0; i < totalMsgs; i++) + (await setupJs.PublishAsync(subject, $"msg-{i}", cancellationToken: cts.Token)).EnsureSuccess(); + + await stream.CreateOrUpdateConsumerAsync(new ConsumerConfig(consumerName), cts.Token); + } + + var nats = new NatsConnection(new NatsOpts { Url = _server.Url }); + var js = new NatsJSContext(nats); + var consumer = await js.GetConsumerAsync(streamName, consumerName, cts.Token); + + var consumeTask = Task.Run( + async () => + { + var count = 0; + await foreach (var msg in consumer.ConsumeAsync(opts: new NatsJSConsumeOpts { MaxMsgs = pullBatch }, cancellationToken: cts.Token)) + { + count++; + await msg.AckAsync(cancellationToken: cts.Token); + await Task.Delay(50, cts.Token); + } + + return count; + }, + cts.Token); + + await Retry.Until( + $"AckFloor >= {bailAt}", + async () => + { + var c = await js.GetConsumerAsync(streamName, consumerName, cts.Token); + return c.Info.AckFloor.ConsumerSeq >= bailAt; + }, + retryDelay: TimeSpan.FromMilliseconds(50), + timeout: TimeSpan.FromSeconds(30)); + + await nats.DisposeAsync(); + + var consumed = await consumeTask; + + // Reader should have drained the entire pulled batch and acked each. + Assert.Equal(pullBatch, consumed); + + await using var check = new NatsConnection(new NatsOpts { Url = _server.Url }); + var checkJs = new NatsJSContext(check); + var info = (await checkJs.GetConsumerAsync(streamName, consumerName, cts.Token)).Info; + + Assert.Equal(0, info.NumAckPending); + Assert.Equal((ulong)pullBatch, (ulong)info.AckFloor.ConsumerSeq); + Assert.Equal((ulong)(totalMsgs - pullBatch), info.NumPending); + } + [Fact] public async Task Consume_stop_test() { diff --git a/tests/NATS.Client.JetStream.Tests/ConsumerFetchTest.cs b/tests/NATS.Client.JetStream.Tests/ConsumerFetchTest.cs index cac69272c..732e480e6 100644 --- a/tests/NATS.Client.JetStream.Tests/ConsumerFetchTest.cs +++ b/tests/NATS.Client.JetStream.Tests/ConsumerFetchTest.cs @@ -81,8 +81,7 @@ public async Task FetchNoWait_test(NatsRequestReplyMode mode) // messages in-flight when DisposeAsync is called must all reach the // channel before it completes. If this test starts flapping, check the // DrainAsync pattern in NatsSubBase and the DisposeAsync overrides in - // NatsJSFetch, NatsJSConsume, NatsJSOrderedConsume. A manual stress - // repro lives under sandbox/SubCleanDispose. + // NatsJSFetch, NatsJSConsume, NatsJSOrderedConsume. [Theory] // TODO: Fix this test From 60aee133836dcf43a5432b1adf5961758f16dcc9 Mon Sep 17 00:00:00 2001 From: Ziya Suzen Date: Sun, 26 Apr 2026 15:04:43 +0100 Subject: [PATCH 06/19] Gate dispose drain behavior behind opt-in options The previous commit changed dispose semantics in two ways: subs are now drained before the socket closes, and consumer dispose waits for in-flight consume loops to finish acking buffered messages. Both are strictly more correct, but the wait can block dispose for up to the configured timeout, which is observable. Add NatsOpts.DrainSubscriptionsOnDispose (bool, default false) for the dispose order, and NatsOpts.ConsumerDrainOnDisposeTimeout (TimeSpan?, default null) for the wait. Defaults preserve current behavior; set both to opt into the graceful path. --- src/NATS.Client.Core/NatsConnection.cs | 20 +++++++++++---- src/NATS.Client.Core/NatsOpts.cs | 25 +++++++++++++++++++ .../Internal/NatsJSConsume.cs | 5 +++- .../Internal/NatsJSFetch.cs | 5 +++- .../Internal/NatsJSOrderedConsume.cs | 5 +++- .../ConsumerConsumeTest.cs | 7 +++++- 6 files changed, 58 insertions(+), 9 deletions(-) diff --git a/src/NATS.Client.Core/NatsConnection.cs b/src/NATS.Client.Core/NatsConnection.cs index 7fee532e7..e23a1e0fa 100644 --- a/src/NATS.Client.Core/NatsConnection.cs +++ b/src/NATS.Client.Core/NatsConnection.cs @@ -289,11 +289,21 @@ public virtual async ValueTask DisposeAsync() #endif } - // Drain subscriptions and flush the writer before tearing down - // the socket so UNSUB/PING/PONG and pending acks can land. - await _subscriptionManager.DisposeAsync().ConfigureAwait(false); - await CommandWriter.DisposeAsync().ConfigureAwait(false); - await DisposeSocketAsync(false).ConfigureAwait(false); + if (Opts.DrainSubscriptionsOnDispose) + { + // Drain subs and flush the writer first so UNSUB/PING/PONG + // and any pending acks can land before the socket closes. + await _subscriptionManager.DisposeAsync().ConfigureAwait(false); + await CommandWriter.DisposeAsync().ConfigureAwait(false); + await DisposeSocketAsync(false).ConfigureAwait(false); + } + else + { + await DisposeSocketAsync(false).ConfigureAwait(false); + await _subscriptionManager.DisposeAsync().ConfigureAwait(false); + await CommandWriter.DisposeAsync().ConfigureAwait(false); + } + _waitForOpenConnection.TrySetCanceled(); #if NET8_0_OR_GREATER await _disposedCts.CancelAsync().ConfigureAwait(false); diff --git a/src/NATS.Client.Core/NatsOpts.cs b/src/NATS.Client.Core/NatsOpts.cs index 9f26b5ec4..c7910b968 100644 --- a/src/NATS.Client.Core/NatsOpts.cs +++ b/src/NATS.Client.Core/NatsOpts.cs @@ -153,6 +153,31 @@ public sealed record NatsOpts public bool WaitUntilSent { get; init; } = false; + /// + /// When true, drains subscriptions + /// and flushes the command writer before closing the socket. (default: false) + /// + /// + /// With the default (false), the socket is closed first, which means UNSUB + /// and any pending writes (e.g. JetStream acks queued by a consume loop) + /// are dropped. Enable to let subscriptions drain cleanly on dispose. + /// Required for to be effective. + /// + public bool DrainSubscriptionsOnDispose { get; init; } = false; + + /// + /// When set, JetStream consumer/fetch dispose waits up to this timeout for + /// the user's consume loop to finish acking buffered messages before + /// returning. (default: null, no wait) + /// + /// + /// Only effective when is also + /// true. Without this wait, messages already delivered into the consumer + /// channel but not yet yielded by the user loop are dropped on dispose + /// and remain NumAckPending until AckWait expires. + /// + public TimeSpan? ConsumerDrainOnDisposeTimeout { get; init; } = null; + /// /// Maximum number of reconnect attempts. (default: -1, unlimited) /// diff --git a/src/NATS.Client.JetStream/Internal/NatsJSConsume.cs b/src/NATS.Client.JetStream/Internal/NatsJSConsume.cs index 5821bd123..08ce8e884 100644 --- a/src/NATS.Client.JetStream/Internal/NatsJSConsume.cs +++ b/src/NATS.Client.JetStream/Internal/NatsJSConsume.cs @@ -314,9 +314,12 @@ protected override async ValueTask WaitForReaderDrainAsync() if (Volatile.Read(ref _readerActive) == 0) return; + if (Connection.Opts.ConsumerDrainOnDisposeTimeout is not { } timeout) + return; + try { - using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + using var cts = new CancellationTokenSource(timeout); await _readerExited.Task.WaitAsync(cts.Token).ConfigureAwait(false); } catch (OperationCanceledException) diff --git a/src/NATS.Client.JetStream/Internal/NatsJSFetch.cs b/src/NATS.Client.JetStream/Internal/NatsJSFetch.cs index 777489ae5..1bd78bb1e 100644 --- a/src/NATS.Client.JetStream/Internal/NatsJSFetch.cs +++ b/src/NATS.Client.JetStream/Internal/NatsJSFetch.cs @@ -225,9 +225,12 @@ protected override async ValueTask WaitForReaderDrainAsync() if (Volatile.Read(ref _readerActive) == 0) return; + if (Connection.Opts.ConsumerDrainOnDisposeTimeout is not { } timeout) + return; + try { - using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + using var cts = new CancellationTokenSource(timeout); await _readerExited.Task.WaitAsync(cts.Token).ConfigureAwait(false); } catch (OperationCanceledException) diff --git a/src/NATS.Client.JetStream/Internal/NatsJSOrderedConsume.cs b/src/NATS.Client.JetStream/Internal/NatsJSOrderedConsume.cs index b23950650..57f6475db 100644 --- a/src/NATS.Client.JetStream/Internal/NatsJSOrderedConsume.cs +++ b/src/NATS.Client.JetStream/Internal/NatsJSOrderedConsume.cs @@ -176,9 +176,12 @@ protected override async ValueTask WaitForReaderDrainAsync() if (Volatile.Read(ref _readerActive) == 0) return; + if (Connection.Opts.ConsumerDrainOnDisposeTimeout is not { } timeout) + return; + try { - using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + using var cts = new CancellationTokenSource(timeout); await _readerExited.Task.WaitAsync(cts.Token).ConfigureAwait(false); } catch (OperationCanceledException) diff --git a/tests/NATS.Client.JetStream.Tests/ConsumerConsumeTest.cs b/tests/NATS.Client.JetStream.Tests/ConsumerConsumeTest.cs index 785594a0d..909a1b5ee 100644 --- a/tests/NATS.Client.JetStream.Tests/ConsumerConsumeTest.cs +++ b/tests/NATS.Client.JetStream.Tests/ConsumerConsumeTest.cs @@ -385,7 +385,12 @@ public async Task Consume_connection_dispose_drains_buffered_messages() await stream.CreateOrUpdateConsumerAsync(new ConsumerConfig(consumerName), cts.Token); } - var nats = new NatsConnection(new NatsOpts { Url = _server.Url }); + var nats = new NatsConnection(new NatsOpts + { + Url = _server.Url, + DrainSubscriptionsOnDispose = true, + ConsumerDrainOnDisposeTimeout = TimeSpan.FromSeconds(30), + }); var js = new NatsJSContext(nats); var consumer = await js.GetConsumerAsync(streamName, consumerName, cts.Token); From 6567ebb28bb3aa2288d8000c108b4126d119ba23 Mon Sep 17 00:00:00 2001 From: Ziya Suzen Date: Sun, 26 Apr 2026 15:34:45 +0100 Subject: [PATCH 07/19] Bump test session timeout to 10 minutes --- tests/xunit.runsettings | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/xunit.runsettings b/tests/xunit.runsettings index d9dcca60a..db71244a6 100644 --- a/tests/xunit.runsettings +++ b/tests/xunit.runsettings @@ -1,7 +1,7 @@  - 300000 + 600000 4.0x From f26a89b904a6b4ed2aa3a9e8b846e12ba492d4ad Mon Sep 17 00:00:00 2001 From: Ziya Suzen Date: Sun, 26 Apr 2026 15:54:43 +0100 Subject: [PATCH 08/19] Fix Consume_connection_dispose flap on slow consumer-stat updates Polling server-side AckFloor to time the dispose was racing the pull consumer's threshold top-up: the reader could process a second batch before the server stat caught up, so the strict consumed == pullBatch check fired Expected 20 / Actual 30 on Linux runners. Use a client-side TaskCompletionSource to signal once the reader has acked bailAt messages, and assert what actually matters: NumAckPending is zero and AckFloor matches the client-side count, regardless of how many batches the consumer pulled. --- .../ConsumerConsumeTest.cs | 24 ++++++++----------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/tests/NATS.Client.JetStream.Tests/ConsumerConsumeTest.cs b/tests/NATS.Client.JetStream.Tests/ConsumerConsumeTest.cs index 909a1b5ee..918a7832d 100644 --- a/tests/NATS.Client.JetStream.Tests/ConsumerConsumeTest.cs +++ b/tests/NATS.Client.JetStream.Tests/ConsumerConsumeTest.cs @@ -394,6 +394,7 @@ public async Task Consume_connection_dispose_drains_buffered_messages() var js = new NatsJSContext(nats); var consumer = await js.GetConsumerAsync(streamName, consumerName, cts.Token); + var reachedBail = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var consumeTask = Task.Run( async () => { @@ -402,6 +403,8 @@ public async Task Consume_connection_dispose_drains_buffered_messages() { count++; await msg.AckAsync(cancellationToken: cts.Token); + if (count >= bailAt) + reachedBail.TrySetResult(); await Task.Delay(50, cts.Token); } @@ -409,30 +412,23 @@ public async Task Consume_connection_dispose_drains_buffered_messages() }, cts.Token); - await Retry.Until( - $"AckFloor >= {bailAt}", - async () => - { - var c = await js.GetConsumerAsync(streamName, consumerName, cts.Token); - return c.Info.AckFloor.ConsumerSeq >= bailAt; - }, - retryDelay: TimeSpan.FromMilliseconds(50), - timeout: TimeSpan.FromSeconds(30)); + await reachedBail.Task.WaitAsync(cts.Token); await nats.DisposeAsync(); var consumed = await consumeTask; - // Reader should have drained the entire pulled batch and acked each. - Assert.Equal(pullBatch, consumed); - await using var check = new NatsConnection(new NatsOpts { Url = _server.Url }); var checkJs = new NatsJSContext(check); var info = (await checkJs.GetConsumerAsync(streamName, consumerName, cts.Token)).Info; + // Whatever the reader processed must all be acked: nothing left + // pending, AckFloor matches the client-side count, NumPending is + // the rest of the stream. + Assert.True(consumed >= bailAt, $"consumed {consumed} should be >= {bailAt}"); Assert.Equal(0, info.NumAckPending); - Assert.Equal((ulong)pullBatch, (ulong)info.AckFloor.ConsumerSeq); - Assert.Equal((ulong)(totalMsgs - pullBatch), info.NumPending); + Assert.Equal((ulong)consumed, (ulong)info.AckFloor.ConsumerSeq); + Assert.Equal((ulong)(totalMsgs - consumed), info.NumPending); } [Fact] From a8eb92c7973e53b1eb9fcb69fd6fd4398c19d3d9 Mon Sep 17 00:00:00 2001 From: Ziya Suzen Date: Sun, 26 Apr 2026 16:29:44 +0100 Subject: [PATCH 09/19] Drain Fetch and ordered consumers on connection dispose Fetch removes itself from the SubscriptionManager via EndSubscription on natural completion (NoMsgs, RequestsPending, MaxBytes, Timeout), so SubscriptionManager.DisposeAsync no longer sees it and the per-sub DrainAsync path that waits for the user reader never fired. Ordered consumer's outer loop tried to RecreateConsumer after the inner cc finished draining, hitting the disposed CommandWriter. Track active reader subs in a per-connection drain participants set, populated from MarkReaderActive / cleared from MarkReaderInactive, and run their DrainAsync from NatsConnection.DisposeAsync between SubscriptionManager.DisposeAsync and CommandWriter.DisposeAsync. Move WaitForReaderDrainAsync outside DrainAsync's _unsubscribed early return so it always runs even when EndSubscription already sent UNSUB. Have the ordered consumer's outer loop bail when the connection is already disposed instead of attempting to recreate. Adds Fetch_connection_dispose_drains_buffered_messages and OrderedConsume_connection_dispose_drains_buffered_messages. --- src/NATS.Client.Core/NatsConnection.cs | 33 +++++++++ src/NATS.Client.Core/NatsSubBase.cs | 72 ++++++++++--------- .../Internal/NatsJSConsume.cs | 9 ++- .../Internal/NatsJSFetch.cs | 9 ++- .../Internal/NatsJSOrderedConsume.cs | 9 ++- .../NatsJSOrderedConsumer.cs | 3 + .../ConsumerFetchTest.cs | 70 ++++++++++++++++++ .../OrderedConsumerTest.cs | 71 ++++++++++++++++++ 8 files changed, 240 insertions(+), 36 deletions(-) diff --git a/src/NATS.Client.Core/NatsConnection.cs b/src/NATS.Client.Core/NatsConnection.cs index e23a1e0fa..2a0c9005f 100644 --- a/src/NATS.Client.Core/NatsConnection.cs +++ b/src/NATS.Client.Core/NatsConnection.cs @@ -48,6 +48,8 @@ public partial class NatsConnection : INatsConnection private readonly ClientOpts _clientOpts; private readonly SubscriptionManager _subscriptionManager; private readonly ReplyTaskFactory _replyTaskFactory; + private readonly object _drainParticipantsGate = new(); + private readonly HashSet _drainParticipants = new(); private ServerInfo? _writableServerInfo; private int _pongCount; @@ -294,6 +296,7 @@ public virtual async ValueTask DisposeAsync() // Drain subs and flush the writer first so UNSUB/PING/PONG // and any pending acks can land before the socket closes. await _subscriptionManager.DisposeAsync().ConfigureAwait(false); + await DrainRemainingParticipantsAsync().ConfigureAwait(false); await CommandWriter.DisposeAsync().ConfigureAwait(false); await DisposeSocketAsync(false).ConfigureAwait(false); } @@ -373,6 +376,18 @@ internal void ResetPongCount() // called only internally internal ValueTask SubscribeCoreAsync(int sid, string subject, string? queueGroup, int? maxMsgs, CancellationToken cancellationToken) => CommandWriter.SubscribeAsync(sid, subject, queueGroup, maxMsgs, cancellationToken); + internal void RegisterDrainParticipant(NatsSubBase sub) + { + lock (_drainParticipantsGate) + _drainParticipants.Add(sub); + } + + internal void UnregisterDrainParticipant(NatsSubBase sub) + { + lock (_drainParticipantsGate) + _drainParticipants.Remove(sub); + } + internal ValueTask UnsubscribeAsync(int sid) { try @@ -1152,4 +1167,22 @@ private void ThrowIfDisposed() if (IsDisposed) throw new ObjectDisposedException(null); } + + // Drain subs that already removed themselves from the SubscriptionManager + // (e.g. JetStream Fetch on natural completion) but still have a user + // iterator reading the channel. Their own DrainAsync waits on the + // reader-exited signal. + private async ValueTask DrainRemainingParticipantsAsync() + { + NatsSubBase[] remaining; + lock (_drainParticipantsGate) + { + if (_drainParticipants.Count == 0) + return; + remaining = _drainParticipants.ToArray(); + } + + foreach (var sub in remaining) + await sub.DrainAsync().ConfigureAwait(false); + } } diff --git a/src/NATS.Client.Core/NatsSubBase.cs b/src/NATS.Client.Core/NatsSubBase.cs index af69cce2d..aa020897b 100644 --- a/src/NATS.Client.Core/NatsSubBase.cs +++ b/src/NATS.Client.Core/NatsSubBase.cs @@ -367,47 +367,53 @@ internal static bool IsHeader503(ReadOnlySequence? headersBuffer) => /// A that represents the asynchronous drain operation. internal async ValueTask DrainAsync() { + var needsUnsub = false; lock (_gate) { - if (_unsubscribed) - return; - _unsubscribed = true; + if (!_unsubscribed) + { + _unsubscribed = true; + needsUnsub = true; + } } - _timeoutTimer?.Change(Timeout.Infinite, Timeout.Infinite); - _idleTimeoutTimer?.Change(Timeout.Infinite, Timeout.Infinite); - _startUpTimeoutTimer?.Change(Timeout.Infinite, Timeout.Infinite); - - // Send UNSUB to the server and remove from subscription manager. - // This stops the server from sending new messages for this subscription. - await _manager.RemoveAsync(this).ConfigureAwait(false); - - // PING/PONG round-trip: after we get the PONG back, we know the server - // has processed our UNSUB and the socket reader has processed all messages - // that were in-flight before the UNSUB was received by the server. - try + if (needsUnsub) { - using var pingCts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); - await Connection.PingAsync(pingCts.Token).ConfigureAwait(false); - } - catch (Exception e) - { - _logger.LogWarning(NatsLogEvents.Subscription, e, "Error during drain ping"); - } + _timeoutTimer?.Change(Timeout.Infinite, Timeout.Infinite); + _idleTimeoutTimer?.Change(Timeout.Infinite, Timeout.Infinite); + _startUpTimeoutTimer?.Change(Timeout.Infinite, Timeout.Infinite); + + // Send UNSUB to the server and remove from subscription manager. + // This stops the server from sending new messages for this subscription. + await _manager.RemoveAsync(this).ConfigureAwait(false); + + // PING/PONG round-trip: after we get the PONG back, we know the server + // has processed our UNSUB and the socket reader has processed all messages + // that were in-flight before the UNSUB was received by the server. + try + { + using var pingCts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await Connection.PingAsync(pingCts.Token).ConfigureAwait(false); + } + catch (Exception e) + { + _logger.LogWarning(NatsLogEvents.Subscription, e, "Error during drain ping"); + } - // Now it's safe to complete the channel, no more messages will arrive. - try - { - TryComplete(); - } - catch (Exception e) - { - _logger.LogWarning(NatsLogEvents.Subscription, e, "Error while completing subscription"); + // Now it's safe to complete the channel, no more messages will arrive. + try + { + TryComplete(); + } + catch (Exception e) + { + _logger.LogWarning(NatsLogEvents.Subscription, e, "Error while completing subscription"); + } } - // Hook for subclasses to wait for an active user reader (e.g. an - // await-foreach loop) to drain the channel before the connection - // tears down the writer/socket. + // Always wait for an active user reader to drain the channel before + // returning, even if the subscription was already ended (e.g. via + // EndSubscription / UnsubscribeAsync) before drain was called. await WaitForReaderDrainAsync().ConfigureAwait(false); } diff --git a/src/NATS.Client.JetStream/Internal/NatsJSConsume.cs b/src/NATS.Client.JetStream/Internal/NatsJSConsume.cs index 08ce8e884..dab2b4463 100644 --- a/src/NATS.Client.JetStream/Internal/NatsJSConsume.cs +++ b/src/NATS.Client.JetStream/Internal/NatsJSConsume.cs @@ -199,11 +199,18 @@ public ValueTask CallMsgNextAsync(string origin, ConsumerGetnextRequest request, public void ResetHeartbeatTimer() => _timer.Change(_hbTimeout, _hbTimeout); - public void MarkReaderActive() => Interlocked.Exchange(ref _readerActive, 1); + public void MarkReaderActive() + { + Interlocked.Exchange(ref _readerActive, 1); + if (Connection is NatsConnection nc) + nc.RegisterDrainParticipant(this); + } public void MarkReaderInactive() { Interlocked.Exchange(ref _readerActive, 0); + if (Connection is NatsConnection nc) + nc.UnregisterDrainParticipant(this); _readerExited.TrySetResult(); } diff --git a/src/NATS.Client.JetStream/Internal/NatsJSFetch.cs b/src/NATS.Client.JetStream/Internal/NatsJSFetch.cs index 1bd78bb1e..66584127c 100644 --- a/src/NATS.Client.JetStream/Internal/NatsJSFetch.cs +++ b/src/NATS.Client.JetStream/Internal/NatsJSFetch.cs @@ -162,11 +162,18 @@ public void ResetHeartbeatTimer() _hbTimer.Change(_hbTimeout, Timeout.Infinite); } - public void MarkReaderActive() => Interlocked.Exchange(ref _readerActive, 1); + public void MarkReaderActive() + { + Interlocked.Exchange(ref _readerActive, 1); + if (Connection is NatsConnection nc) + nc.RegisterDrainParticipant(this); + } public void MarkReaderInactive() { Interlocked.Exchange(ref _readerActive, 0); + if (Connection is NatsConnection nc) + nc.UnregisterDrainParticipant(this); _readerExited.TrySetResult(); } diff --git a/src/NATS.Client.JetStream/Internal/NatsJSOrderedConsume.cs b/src/NATS.Client.JetStream/Internal/NatsJSOrderedConsume.cs index 57f6475db..dc896c2b1 100644 --- a/src/NATS.Client.JetStream/Internal/NatsJSOrderedConsume.cs +++ b/src/NATS.Client.JetStream/Internal/NatsJSOrderedConsume.cs @@ -134,11 +134,18 @@ public ValueTask CallMsgNextAsync(string origin, ConsumerGetnextRequest request, public void ResetHeartbeatTimer() => _timer.Change(_hbTimeout, Timeout.Infinite); - public void MarkReaderActive() => Interlocked.Exchange(ref _readerActive, 1); + public void MarkReaderActive() + { + Interlocked.Exchange(ref _readerActive, 1); + if (Connection is NatsConnection nc) + nc.RegisterDrainParticipant(this); + } public void MarkReaderInactive() { Interlocked.Exchange(ref _readerActive, 0); + if (Connection is NatsConnection nc) + nc.UnregisterDrainParticipant(this); _readerExited.TrySetResult(); } diff --git a/src/NATS.Client.JetStream/NatsJSOrderedConsumer.cs b/src/NATS.Client.JetStream/NatsJSOrderedConsumer.cs index 023d04286..769853ca9 100644 --- a/src/NATS.Client.JetStream/NatsJSOrderedConsumer.cs +++ b/src/NATS.Client.JetStream/NatsJSOrderedConsumer.cs @@ -76,6 +76,9 @@ public async IAsyncEnumerable> ConsumeAsync( ulong seq = 0; while (!cancellationToken.IsCancellationRequested) { + if (_context.Connection is NatsConnection { IsDisposed: true }) + yield break; + var consumer = await RecreateConsumer(consumerName, seq, cancellationToken); consumerName = consumer.Info.Name; _logger.LogInformation(NatsJSLogEvents.NewConsumer, "Created {ConsumerName} with sequence {Seq}", consumerName, seq); diff --git a/tests/NATS.Client.JetStream.Tests/ConsumerFetchTest.cs b/tests/NATS.Client.JetStream.Tests/ConsumerFetchTest.cs index 732e480e6..644aaaad2 100644 --- a/tests/NATS.Client.JetStream.Tests/ConsumerFetchTest.cs +++ b/tests/NATS.Client.JetStream.Tests/ConsumerFetchTest.cs @@ -1,5 +1,6 @@ using NATS.Client.Core.Tests; using NATS.Client.Core2.Tests; +using NATS.Client.JetStream.Models; namespace NATS.Client.JetStream.Tests; @@ -170,4 +171,73 @@ await Retry.Until( await consumer.RefreshAsync(cts.Token); Assert.Equal(0, consumer.Info.NumAckPending); } + + [Fact] + public async Task Fetch_connection_dispose_drains_buffered_messages() + { + const int totalMsgs = 100; + const int pullBatch = 20; + const int bailAt = 10; + + var prefix = _server.GetNextId(); + var streamName = $"{prefix}s1"; + var subject = $"{prefix}s1.x"; + var consumerName = $"{prefix}c1"; + + var cts = new CancellationTokenSource(TimeSpan.FromSeconds(60)); + + await using (var setup = new NatsConnection(new NatsOpts { Url = _server.Url })) + { + var setupJs = new NatsJSContext(setup); + var stream = await setupJs.CreateStreamAsync(new StreamConfig(streamName, [subject]), cts.Token); + + for (var i = 0; i < totalMsgs; i++) + (await setupJs.PublishAsync(subject, $"msg-{i}", cancellationToken: cts.Token)).EnsureSuccess(); + + await stream.CreateOrUpdateConsumerAsync(new ConsumerConfig(consumerName), cts.Token); + } + + var nats = new NatsConnection(new NatsOpts + { + Url = _server.Url, + DrainSubscriptionsOnDispose = true, + ConsumerDrainOnDisposeTimeout = TimeSpan.FromSeconds(30), + }); + var js = new NatsJSContext(nats); + var consumer = await js.GetConsumerAsync(streamName, consumerName, cts.Token); + + var reachedBail = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var fetchTask = Task.Run( + async () => + { + var count = 0; + var fetchOpts = new NatsJSFetchOpts { MaxMsgs = pullBatch, Expires = TimeSpan.FromSeconds(30) }; + await foreach (var msg in consumer.FetchAsync(fetchOpts, cancellationToken: cts.Token)) + { + count++; + await msg.AckAsync(cancellationToken: cts.Token); + if (count >= bailAt) + reachedBail.TrySetResult(); + await Task.Delay(50, cts.Token); + } + + return count; + }, + cts.Token); + + await reachedBail.Task.WaitAsync(cts.Token); + + await nats.DisposeAsync(); + + var consumed = await fetchTask; + + await using var check = new NatsConnection(new NatsOpts { Url = _server.Url }); + var checkJs = new NatsJSContext(check); + var info = (await checkJs.GetConsumerAsync(streamName, consumerName, cts.Token)).Info; + + Assert.True(consumed >= bailAt, $"consumed {consumed} should be >= {bailAt}"); + Assert.Equal(0, info.NumAckPending); + Assert.Equal((ulong)consumed, (ulong)info.AckFloor.ConsumerSeq); + Assert.Equal((ulong)(totalMsgs - consumed), info.NumPending); + } } diff --git a/tests/NATS.Client.JetStream.Tests/OrderedConsumerTest.cs b/tests/NATS.Client.JetStream.Tests/OrderedConsumerTest.cs index 85235fffb..dd240f95b 100644 --- a/tests/NATS.Client.JetStream.Tests/OrderedConsumerTest.cs +++ b/tests/NATS.Client.JetStream.Tests/OrderedConsumerTest.cs @@ -494,4 +494,75 @@ public async Task Fetch_maxbytes_with_consumer_deletion() _output.WriteLine($"Received {count} messages with MaxBytes mode"); Assert.True(count > 5, "Should have received messages after consumer deletion"); } + + [Fact] + public async Task OrderedConsume_connection_dispose_drains_buffered_messages() + { + const int totalMsgs = 100; + const int pullBatch = 20; + const int bailAt = 10; + + var prefix = _server.GetNextId(); + var streamName = $"{prefix}s1"; + var subject = $"{prefix}s1.x"; + + var cts = new CancellationTokenSource(TimeSpan.FromSeconds(60)); + + await using (var setup = new NatsConnection(new NatsOpts { Url = _server.Url })) + { + var setupJs = new NatsJSContext(setup); + await setupJs.CreateStreamAsync(new StreamConfig(streamName, [subject]), cts.Token); + + for (var i = 0; i < totalMsgs; i++) + (await setupJs.PublishAsync(subject, i, cancellationToken: cts.Token)).EnsureSuccess(); + } + + var nats = new NatsConnection(new NatsOpts + { + Url = _server.Url, + DrainSubscriptionsOnDispose = true, + ConsumerDrainOnDisposeTimeout = TimeSpan.FromSeconds(30), + }); + var js = new NatsJSContext(nats); + var stream = await js.GetStreamAsync(streamName, cancellationToken: cts.Token); + var consumer = await stream.CreateOrderedConsumerAsync(cancellationToken: cts.Token); + + var reachedBail = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var lastSeq = -1; + Exception? readerError = null; + var consumeTask = Task.Run( + async () => + { + var count = 0; + try + { + var consumeOpts = new NatsJSConsumeOpts { MaxMsgs = pullBatch, Expires = TimeSpan.FromSeconds(30) }; + await foreach (var msg in consumer.ConsumeAsync(opts: consumeOpts, cancellationToken: cts.Token)) + { + Volatile.Write(ref lastSeq, msg.Data); + count++; + if (count >= bailAt) + reachedBail.TrySetResult(); + await Task.Delay(50, cts.Token); + } + } + catch (Exception e) when (e is not OperationCanceledException) + { + readerError = e; + } + + return count; + }, + cts.Token); + + await reachedBail.Task.WaitAsync(cts.Token); + + await nats.DisposeAsync(); + + var consumed = await consumeTask; + + Assert.Null(readerError); + Assert.True(consumed >= bailAt, $"consumed {consumed} should be >= {bailAt}"); + Assert.Equal(consumed - 1, Volatile.Read(ref lastSeq)); + } } From e7421ed2380e9735ab6d86b6a13111c604add80c Mon Sep 17 00:00:00 2001 From: Ziya Suzen Date: Mon, 27 Apr 2026 12:29:21 +0100 Subject: [PATCH 10/19] docs: add blog with dispose-drain announcement --- .../blog/2026-04-27-drain-on-dispose.md | 40 +++++++++++++++++++ tools/site_src/blog/index.md | 5 +++ tools/site_src/blog/toc.yml | 2 + tools/site_src/docfx.json | 2 + tools/site_src/toc.yml | 4 ++ 5 files changed, 53 insertions(+) create mode 100644 tools/site_src/blog/2026-04-27-drain-on-dispose.md create mode 100644 tools/site_src/blog/index.md create mode 100644 tools/site_src/blog/toc.yml diff --git a/tools/site_src/blog/2026-04-27-drain-on-dispose.md b/tools/site_src/blog/2026-04-27-drain-on-dispose.md new file mode 100644 index 000000000..61ac9d78e --- /dev/null +++ b/tools/site_src/blog/2026-04-27-drain-on-dispose.md @@ -0,0 +1,40 @@ +--- +title: Opt-in drain on dispose for JetStream consumers +date: 2026-04-27 +--- + +# Opt-in drain on dispose for JetStream consumers + +_2026-04-27_ + +Disposing a `NatsConnection` while a JetStream `Consume`, `Fetch`, or +ordered-consumer loop was still buffering messages used to drop pending +acks. The socket got closed before the command writer flushed, so the +`UNSUB` and any acks the user loop had queued went nowhere. The messages +stayed `NumAckPending` on the server until `AckWait` expired and were +then redelivered. + +There are now two opt-in options on `NatsOpts` to control this: + +- `DrainSubscriptionsOnDispose` (default `false`). On dispose, drain + subscriptions and flush the command writer before closing the socket. +- `ConsumerDrainOnDisposeTimeout` (default `null`). When set, JetStream + consumer and fetch dispose waits up to this timeout for the user's + consume loop to finish acking the buffered messages before returning. + Only effective when `DrainSubscriptionsOnDispose` is also `true`. + +```csharp +var opts = NatsOpts.Default with +{ + DrainSubscriptionsOnDispose = true, + ConsumerDrainOnDisposeTimeout = TimeSpan.FromSeconds(5), +}; + +await using var nats = new NatsConnection(opts); +``` + +Both default to off, so existing apps see no behavior change. Turn them +on if you want a clean shutdown without redeliveries. + +Applies to `Consume`, `Fetch`, and ordered consumer loops in +`NATS.Client.JetStream`. Lands in `2.8.0`. diff --git a/tools/site_src/blog/index.md b/tools/site_src/blog/index.md new file mode 100644 index 000000000..3be7f64be --- /dev/null +++ b/tools/site_src/blog/index.md @@ -0,0 +1,5 @@ +# Blog + +Announcements and short notes about NATS .NET. + +- [Opt-in drain on dispose for JetStream consumers](2026-04-27-drain-on-dispose.md) (2026-04-27) diff --git a/tools/site_src/blog/toc.yml b/tools/site_src/blog/toc.yml new file mode 100644 index 000000000..dfb8e6fa9 --- /dev/null +++ b/tools/site_src/blog/toc.yml @@ -0,0 +1,2 @@ +- name: Opt-in drain on dispose for JetStream consumers (2026-04-27) + href: 2026-04-27-drain-on-dispose.md diff --git a/tools/site_src/docfx.json b/tools/site_src/docfx.json index 79189d8cd..578493469 100644 --- a/tools/site_src/docfx.json +++ b/tools/site_src/docfx.json @@ -29,6 +29,8 @@ "files": [ "documentation/**.md", "documentation/**/toc.yml", + "blog/**.md", + "blog/**/toc.yml", "toc.yml", "*.md" ], diff --git a/tools/site_src/toc.yml b/tools/site_src/toc.yml index b49e5aebc..9b91ec115 100644 --- a/tools/site_src/toc.yml +++ b/tools/site_src/toc.yml @@ -5,6 +5,10 @@ href: api/ homepage: api/index.md +- name: Blog + href: blog/ + homepage: blog/index.md + - name: GitHub href: https://github.com/nats-io/nats.net From 8d68a6ba4777ae5fab5149197069c75cc612d364 Mon Sep 17 00:00:00 2001 From: Ziya Suzen Date: Mon, 27 Apr 2026 12:43:07 +0100 Subject: [PATCH 11/19] docs: add resilience note to dispose-drain post --- tools/site_src/blog/2026-04-27-drain-on-dispose.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tools/site_src/blog/2026-04-27-drain-on-dispose.md b/tools/site_src/blog/2026-04-27-drain-on-dispose.md index 61ac9d78e..090f6906c 100644 --- a/tools/site_src/blog/2026-04-27-drain-on-dispose.md +++ b/tools/site_src/blog/2026-04-27-drain-on-dispose.md @@ -36,5 +36,14 @@ await using var nats = new NatsConnection(opts); Both default to off, so existing apps see no behavior change. Turn them on if you want a clean shutdown without redeliveries. +## A note on resilience + +You do not lose messages either way. JetStream redelivers anything that +wasn't acked once `AckWait` expires, which is the whole point of +at-least-once delivery. The old default is still correct, just noisier: +a graceful shutdown can cause redeliveries you didn't actually need. +For most apps that's fine. These options are for when you want shutdown +to be precise rather than just safe. + Applies to `Consume`, `Fetch`, and ordered consumer loops in `NATS.Client.JetStream`. Lands in `2.8.0`. From 669317a42de539581f841e31ab142f8e5d53d453 Mon Sep 17 00:00:00 2001 From: Ziya Suzen Date: Tue, 28 Apr 2026 11:44:12 +0100 Subject: [PATCH 12/19] Tighten DrainAsync exception handling --- src/NATS.Client.Core/NatsSubBase.cs | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/src/NATS.Client.Core/NatsSubBase.cs b/src/NATS.Client.Core/NatsSubBase.cs index aa020897b..e5476fbfc 100644 --- a/src/NATS.Client.Core/NatsSubBase.cs +++ b/src/NATS.Client.Core/NatsSubBase.cs @@ -395,20 +395,13 @@ internal async ValueTask DrainAsync() using var pingCts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); await Connection.PingAsync(pingCts.Token).ConfigureAwait(false); } - catch (Exception e) + catch (OperationCanceledException) { - _logger.LogWarning(NatsLogEvents.Subscription, e, "Error during drain ping"); + // 5s timeout reached or connection cancelled mid-dispose. Expected on dispose path. } // Now it's safe to complete the channel, no more messages will arrive. - try - { - TryComplete(); - } - catch (Exception e) - { - _logger.LogWarning(NatsLogEvents.Subscription, e, "Error while completing subscription"); - } + TryComplete(); } // Always wait for an active user reader to drain the channel before From 08970c24e5e9cf0f358c42d8117fd1dc757edef3 Mon Sep 17 00:00:00 2001 From: Ziya Suzen Date: Tue, 28 Apr 2026 12:03:53 +0100 Subject: [PATCH 13/19] Gate consumer dispose drain on DrainSubscriptionsOnDispose --- src/NATS.Client.Core/NatsSubBase.cs | 9 +++++++++ src/NATS.Client.JetStream/Internal/NatsJSConsume.cs | 4 ++-- src/NATS.Client.JetStream/Internal/NatsJSFetch.cs | 4 ++-- .../Internal/NatsJSOrderedConsume.cs | 4 ++-- tests/NATS.Client.JetStream.Tests/ConsumerConsumeTest.cs | 6 +++++- tests/NATS.Client.JetStream.Tests/ConsumerFetchTest.cs | 7 ++++++- 6 files changed, 26 insertions(+), 8 deletions(-) diff --git a/src/NATS.Client.Core/NatsSubBase.cs b/src/NATS.Client.Core/NatsSubBase.cs index e5476fbfc..1fabdbc52 100644 --- a/src/NATS.Client.Core/NatsSubBase.cs +++ b/src/NATS.Client.Core/NatsSubBase.cs @@ -364,9 +364,18 @@ internal static bool IsHeader503(ReadOnlySequence? headersBuffer) => /// This avoids the race where TryComplete() closes the channel while /// the socket reader is still delivering messages. /// + /// + /// No-op unless is + /// enabled. The connection-level dispose path also gates entry into + /// this method on the same flag (see ordering in + /// ). + /// /// A that represents the asynchronous drain operation. internal async ValueTask DrainAsync() { + if (!Connection.Opts.DrainSubscriptionsOnDispose) + return; + var needsUnsub = false; lock (_gate) { diff --git a/src/NATS.Client.JetStream/Internal/NatsJSConsume.cs b/src/NATS.Client.JetStream/Internal/NatsJSConsume.cs index dab2b4463..8a6a667ce 100644 --- a/src/NATS.Client.JetStream/Internal/NatsJSConsume.cs +++ b/src/NATS.Client.JetStream/Internal/NatsJSConsume.cs @@ -237,8 +237,8 @@ public override async ValueTask DisposeAsync() Interlocked.Exchange(ref _disposed, 1); try { - // Drain: UNSUB -> PING/PONG -> TryComplete, so all in-flight - // messages are written to the channel before it's completed. + // Drain (UNSUB -> PING/PONG -> TryComplete) is no-op unless + // DrainSubscriptionsOnDispose is enabled. await DrainAsync().ConfigureAwait(false); await base.DisposeAsync().ConfigureAwait(false); } diff --git a/src/NATS.Client.JetStream/Internal/NatsJSFetch.cs b/src/NATS.Client.JetStream/Internal/NatsJSFetch.cs index 66584127c..e107e0bdd 100644 --- a/src/NATS.Client.JetStream/Internal/NatsJSFetch.cs +++ b/src/NATS.Client.JetStream/Internal/NatsJSFetch.cs @@ -182,8 +182,8 @@ public override async ValueTask DisposeAsync() Interlocked.Exchange(ref _disposed, 1); try { - // Drain: UNSUB -> PING/PONG -> TryComplete, so all in-flight - // messages are written to the channel before it's completed. + // Drain (UNSUB -> PING/PONG -> TryComplete) is no-op unless + // DrainSubscriptionsOnDispose is enabled. await DrainAsync().ConfigureAwait(false); await base.DisposeAsync().ConfigureAwait(false); } diff --git a/src/NATS.Client.JetStream/Internal/NatsJSOrderedConsume.cs b/src/NATS.Client.JetStream/Internal/NatsJSOrderedConsume.cs index dc896c2b1..f60b878f9 100644 --- a/src/NATS.Client.JetStream/Internal/NatsJSOrderedConsume.cs +++ b/src/NATS.Client.JetStream/Internal/NatsJSOrderedConsume.cs @@ -156,8 +156,8 @@ public override async ValueTask DisposeAsync() _context.Connection.ConnectionDisconnected -= ConnectionOnConnectionDisconnected; try { - // Drain: UNSUB -> PING/PONG -> TryComplete, so all in-flight - // messages are written to the channel before it's completed. + // Drain (UNSUB -> PING/PONG -> TryComplete) is no-op unless + // DrainSubscriptionsOnDispose is enabled. await DrainAsync().ConfigureAwait(false); await base.DisposeAsync().ConfigureAwait(false); } diff --git a/tests/NATS.Client.JetStream.Tests/ConsumerConsumeTest.cs b/tests/NATS.Client.JetStream.Tests/ConsumerConsumeTest.cs index 918a7832d..1b2fbc7a7 100644 --- a/tests/NATS.Client.JetStream.Tests/ConsumerConsumeTest.cs +++ b/tests/NATS.Client.JetStream.Tests/ConsumerConsumeTest.cs @@ -277,7 +277,11 @@ await Retry.Until( [Fact] public async Task Consume_dispose_test() { - await using var nats = new NatsConnection(new NatsOpts { Url = _server.Url }); + await using var nats = new NatsConnection(new NatsOpts + { + Url = _server.Url, + DrainSubscriptionsOnDispose = true, + }); var prefix = _server.GetNextId(); var cts = new CancellationTokenSource(TimeSpan.FromSeconds(60)); diff --git a/tests/NATS.Client.JetStream.Tests/ConsumerFetchTest.cs b/tests/NATS.Client.JetStream.Tests/ConsumerFetchTest.cs index 644aaaad2..6804aebac 100644 --- a/tests/NATS.Client.JetStream.Tests/ConsumerFetchTest.cs +++ b/tests/NATS.Client.JetStream.Tests/ConsumerFetchTest.cs @@ -90,7 +90,12 @@ public async Task FetchNoWait_test(NatsRequestReplyMode mode) [InlineData(NatsRequestReplyMode.SharedInbox)] public async Task Fetch_dispose_test(NatsRequestReplyMode mode) { - await using var nats = new NatsConnection(new NatsOpts { Url = _server.Url, RequestReplyMode = mode }); + await using var nats = new NatsConnection(new NatsOpts + { + Url = _server.Url, + RequestReplyMode = mode, + DrainSubscriptionsOnDispose = true, + }); var prefix = _server.GetNextId(); var cts = new CancellationTokenSource(TimeSpan.FromSeconds(60)); From 80e6692e4847f73b72f61864561897d910baa451 Mon Sep 17 00:00:00 2001 From: Ziya Suzen Date: Tue, 28 Apr 2026 12:24:39 +0100 Subject: [PATCH 14/19] Document drain mechanism on DrainSubscriptionsOnDispose [no ci] --- src/NATS.Client.Core/NatsOpts.cs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/NATS.Client.Core/NatsOpts.cs b/src/NATS.Client.Core/NatsOpts.cs index c7910b968..bfffca9d5 100644 --- a/src/NATS.Client.Core/NatsOpts.cs +++ b/src/NATS.Client.Core/NatsOpts.cs @@ -158,10 +158,21 @@ public sealed record NatsOpts /// and flushes the command writer before closing the socket. (default: false) /// /// + /// /// With the default (false), the socket is closed first, which means UNSUB /// and any pending writes (e.g. JetStream acks queued by a consume loop) /// are dropped. Enable to let subscriptions drain cleanly on dispose. /// Required for to be effective. + /// + /// + /// Drain mechanism: each subscription sends UNSUB, then a PING and waits + /// for the PONG (up to 5 seconds). Once the PONG arrives the server has + /// processed UNSUB and the socket reader has delivered any messages the + /// server sent before that, so completing the user channel afterwards + /// won't drop in-flight messages. Dispose therefore blocks for at most + /// one round-trip per drained subscription, plus the + /// wait if set. + /// /// public bool DrainSubscriptionsOnDispose { get; init; } = false; From f8efb0a446a1219714bc405558b9ba348abe4f59 Mon Sep 17 00:00:00 2001 From: Ziya Suzen Date: Tue, 28 Apr 2026 12:38:22 +0100 Subject: [PATCH 15/19] Hoist drain-participant state into NatsSubBase --- src/NATS.Client.Core/NatsSubBase.cs | 56 ++++++++++++++++++- .../Internal/NatsJSConsume.cs | 38 ------------- .../Internal/NatsJSFetch.cs | 38 ------------- .../Internal/NatsJSOrderedConsume.cs | 38 ------------- .../ConsumerFetchTest.cs | 7 +-- 5 files changed, 55 insertions(+), 122 deletions(-) diff --git a/src/NATS.Client.Core/NatsSubBase.cs b/src/NATS.Client.Core/NatsSubBase.cs index 1fabdbc52..2ff14538b 100644 --- a/src/NATS.Client.Core/NatsSubBase.cs +++ b/src/NATS.Client.Core/NatsSubBase.cs @@ -50,6 +50,7 @@ public abstract class NatsSubBase private int _pendingMsgs; private Exception? _exception; private int _isSlowConsumer; + private TaskCompletionSource? _readerExited; /// /// Creates a new instance of . @@ -438,6 +439,33 @@ internal async ValueTask DrainAsync() /// ValueTask internal virtual ValueTask WriteReconnectCommandsAsync(CommandWriter commandWriter, int sid) => commandWriter.SubscribeAsync(sid, Subject, QueueGroup, PendingMsgs, CancellationToken.None); + /// + /// Marks the user-side reader as active so dispose can wait for it to + /// drain buffered messages before completing. Pairs with + /// . + /// + internal void MarkReaderActive() + { + if (_readerExited is null) + { + var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + Interlocked.CompareExchange(ref _readerExited, tcs, null); + } + + if (Connection is NatsConnection nc) + nc.RegisterDrainParticipant(this); + } + + /// + /// Signals the user-side reader has exited. Pairs with . + /// + internal void MarkReaderInactive() + { + if (Connection is NatsConnection nc) + nc.UnregisterDrainParticipant(this); + _readerExited?.TrySetResult(); + } + /// /// Invoked when a MSG or HMSG arrives for the subscription. /// @@ -452,10 +480,19 @@ internal async ValueTask DrainAsync() protected abstract ValueTask ReceiveInternalAsync(string subject, string? replyTo, ReadOnlySequence? headersBuffer, ReadOnlySequence payloadBuffer); /// - /// Subclasses override to wait for the user-side reader to finish - /// draining buffered messages. Default is no-op. + /// Waits for the user-side reader to exit so buffered messages can be + /// drained on dispose. No-op when the reader was never marked active or + /// when is unset. /// - protected virtual ValueTask WaitForReaderDrainAsync() => default; + protected virtual ValueTask WaitForReaderDrainAsync() + { + var tcs = _readerExited; + if (tcs is null || tcs.Task.IsCompleted) + return default; + if (Connection.Opts.ConsumerDrainOnDisposeTimeout is not { } timeout) + return default; + return WaitForReaderDrainCoreAsync(tcs.Task, timeout); + } /// /// Resets the slow consumer state if the channel has drained, allowing another @@ -548,4 +585,17 @@ protected void EndSubscription(NatsSubEndReason reason) #pragma warning restore VSTHRD110 #pragma warning restore CA2012 } + + private async ValueTask WaitForReaderDrainCoreAsync(Task task, TimeSpan timeout) + { + try + { + using var cts = new CancellationTokenSource(timeout); + await task.WaitAsync(cts.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + _logger.LogWarning(NatsLogEvents.Subscription, "Timeout waiting for subscription reader to exit on dispose"); + } + } } diff --git a/src/NATS.Client.JetStream/Internal/NatsJSConsume.cs b/src/NATS.Client.JetStream/Internal/NatsJSConsume.cs index 8a6a667ce..0067aba4d 100644 --- a/src/NATS.Client.JetStream/Internal/NatsJSConsume.cs +++ b/src/NATS.Client.JetStream/Internal/NatsJSConsume.cs @@ -43,12 +43,10 @@ internal class NatsJSConsume : NatsSubBase private readonly int _maxConsecutive503Errors; private readonly object _pendingGate = new(); - private readonly TaskCompletionSource _readerExited = new(TaskCreationOptions.RunContinuationsAsynchronously); private long _pendingMsgs; private long _pendingBytes; private int _disposed; private int _consecutive503Errors; - private int _readerActive; public NatsJSConsume( long maxMsgs, @@ -199,21 +197,6 @@ public ValueTask CallMsgNextAsync(string origin, ConsumerGetnextRequest request, public void ResetHeartbeatTimer() => _timer.Change(_hbTimeout, _hbTimeout); - public void MarkReaderActive() - { - Interlocked.Exchange(ref _readerActive, 1); - if (Connection is NatsConnection nc) - nc.RegisterDrainParticipant(this); - } - - public void MarkReaderInactive() - { - Interlocked.Exchange(ref _readerActive, 0); - if (Connection is NatsConnection nc) - nc.UnregisterDrainParticipant(this); - _readerExited.TrySetResult(); - } - public void Delivered(int msgSize) { lock (_pendingGate) @@ -237,8 +220,6 @@ public override async ValueTask DisposeAsync() Interlocked.Exchange(ref _disposed, 1); try { - // Drain (UNSUB -> PING/PONG -> TryComplete) is no-op unless - // DrainSubscriptionsOnDispose is enabled. await DrainAsync().ConfigureAwait(false); await base.DisposeAsync().ConfigureAwait(false); } @@ -316,25 +297,6 @@ await commandWriter.PublishAsync( } } - protected override async ValueTask WaitForReaderDrainAsync() - { - if (Volatile.Read(ref _readerActive) == 0) - return; - - if (Connection.Opts.ConsumerDrainOnDisposeTimeout is not { } timeout) - return; - - try - { - using var cts = new CancellationTokenSource(timeout); - await _readerExited.Task.WaitAsync(cts.Token).ConfigureAwait(false); - } - catch (OperationCanceledException) - { - _logger.LogWarning(NatsJSLogEvents.Internal, "Timeout waiting for consumer reader to exit on dispose"); - } - } - protected override async ValueTask ReceiveInternalAsync( string subject, string? replyTo, diff --git a/src/NATS.Client.JetStream/Internal/NatsJSFetch.cs b/src/NATS.Client.JetStream/Internal/NatsJSFetch.cs index e107e0bdd..26ecd16da 100644 --- a/src/NATS.Client.JetStream/Internal/NatsJSFetch.cs +++ b/src/NATS.Client.JetStream/Internal/NatsJSFetch.cs @@ -30,11 +30,9 @@ internal class NatsJSFetch : NatsSubBase private readonly TimeSpan _idle; private readonly long _hbTimeout; - private readonly TaskCompletionSource _readerExited = new(TaskCreationOptions.RunContinuationsAsynchronously); private long _pendingMsgs; private long _pendingBytes; private int _disposed; - private int _readerActive; public NatsJSFetch( long maxMsgs, @@ -162,28 +160,11 @@ public void ResetHeartbeatTimer() _hbTimer.Change(_hbTimeout, Timeout.Infinite); } - public void MarkReaderActive() - { - Interlocked.Exchange(ref _readerActive, 1); - if (Connection is NatsConnection nc) - nc.RegisterDrainParticipant(this); - } - - public void MarkReaderInactive() - { - Interlocked.Exchange(ref _readerActive, 0); - if (Connection is NatsConnection nc) - nc.UnregisterDrainParticipant(this); - _readerExited.TrySetResult(); - } - public override async ValueTask DisposeAsync() { Interlocked.Exchange(ref _disposed, 1); try { - // Drain (UNSUB -> PING/PONG -> TryComplete) is no-op unless - // DrainSubscriptionsOnDispose is enabled. await DrainAsync().ConfigureAwait(false); await base.DisposeAsync().ConfigureAwait(false); } @@ -227,25 +208,6 @@ await commandWriter.PublishAsync( cancellationToken: CancellationToken.None); } - protected override async ValueTask WaitForReaderDrainAsync() - { - if (Volatile.Read(ref _readerActive) == 0) - return; - - if (Connection.Opts.ConsumerDrainOnDisposeTimeout is not { } timeout) - return; - - try - { - using var cts = new CancellationTokenSource(timeout); - await _readerExited.Task.WaitAsync(cts.Token).ConfigureAwait(false); - } - catch (OperationCanceledException) - { - _logger.LogWarning(NatsJSLogEvents.Internal, "Timeout waiting for fetch reader to exit on dispose"); - } - } - protected override async ValueTask ReceiveInternalAsync( string subject, string? replyTo, diff --git a/src/NATS.Client.JetStream/Internal/NatsJSOrderedConsume.cs b/src/NATS.Client.JetStream/Internal/NatsJSOrderedConsume.cs index f60b878f9..78d61cd97 100644 --- a/src/NATS.Client.JetStream/Internal/NatsJSOrderedConsume.cs +++ b/src/NATS.Client.JetStream/Internal/NatsJSOrderedConsume.cs @@ -32,11 +32,9 @@ internal class NatsJSOrderedConsume : NatsSubBase private readonly long _thresholdBytes; private readonly object _pendingGate = new(); - private readonly TaskCompletionSource _readerExited = new(TaskCreationOptions.RunContinuationsAsynchronously); private long _pendingMsgs; private long _pendingBytes; private int _disposed; - private int _readerActive; public NatsJSOrderedConsume( long maxMsgs, @@ -134,21 +132,6 @@ public ValueTask CallMsgNextAsync(string origin, ConsumerGetnextRequest request, public void ResetHeartbeatTimer() => _timer.Change(_hbTimeout, Timeout.Infinite); - public void MarkReaderActive() - { - Interlocked.Exchange(ref _readerActive, 1); - if (Connection is NatsConnection nc) - nc.RegisterDrainParticipant(this); - } - - public void MarkReaderInactive() - { - Interlocked.Exchange(ref _readerActive, 0); - if (Connection is NatsConnection nc) - nc.UnregisterDrainParticipant(this); - _readerExited.TrySetResult(); - } - public override async ValueTask DisposeAsync() { Interlocked.Exchange(ref _disposed, 1); @@ -156,8 +139,6 @@ public override async ValueTask DisposeAsync() _context.Connection.ConnectionDisconnected -= ConnectionOnConnectionDisconnected; try { - // Drain (UNSUB -> PING/PONG -> TryComplete) is no-op unless - // DrainSubscriptionsOnDispose is enabled. await DrainAsync().ConfigureAwait(false); await base.DisposeAsync().ConfigureAwait(false); } @@ -178,25 +159,6 @@ internal override ValueTask WriteReconnectCommandsAsync(CommandWriter commandWri return default; } - protected override async ValueTask WaitForReaderDrainAsync() - { - if (Volatile.Read(ref _readerActive) == 0) - return; - - if (Connection.Opts.ConsumerDrainOnDisposeTimeout is not { } timeout) - return; - - try - { - using var cts = new CancellationTokenSource(timeout); - await _readerExited.Task.WaitAsync(cts.Token).ConfigureAwait(false); - } - catch (OperationCanceledException) - { - _logger.LogWarning(NatsJSLogEvents.Internal, "Timeout waiting for ordered-consume reader to exit on dispose"); - } - } - protected override async ValueTask ReceiveInternalAsync( string subject, string? replyTo, diff --git a/tests/NATS.Client.JetStream.Tests/ConsumerFetchTest.cs b/tests/NATS.Client.JetStream.Tests/ConsumerFetchTest.cs index 6804aebac..c67d7f124 100644 --- a/tests/NATS.Client.JetStream.Tests/ConsumerFetchTest.cs +++ b/tests/NATS.Client.JetStream.Tests/ConsumerFetchTest.cs @@ -78,11 +78,8 @@ public async Task FetchNoWait_test(NatsRequestReplyMode mode) Assert.Equal(10, count); } - // Canary for the dispose race in JetStream fetch/consume/ordered-consume: - // messages in-flight when DisposeAsync is called must all reach the - // channel before it completes. If this test starts flapping, check the - // DrainAsync pattern in NatsSubBase and the DisposeAsync overrides in - // NatsJSFetch, NatsJSConsume, NatsJSOrderedConsume. + // Canary for the dispose race: messages in-flight when DisposeAsync is + // called must all reach the channel before it completes. [Theory] // TODO: Fix this test From 9dae4c90ef22a0ebdd12dbc1e7dd026bbdeed5b1 Mon Sep 17 00:00:00 2001 From: Ziya Suzen Date: Tue, 28 Apr 2026 13:15:35 +0100 Subject: [PATCH 16/19] Parallelise drain participants and add DrainPingTimeout opt --- src/NATS.Client.Core/NatsConnection.cs | 14 +++++++---- src/NATS.Client.Core/NatsOpts.cs | 25 ++++++++++++++----- src/NATS.Client.Core/NatsSubBase.cs | 4 +-- .../blog/2026-04-27-drain-on-dispose.md | 3 +++ 4 files changed, 33 insertions(+), 13 deletions(-) diff --git a/src/NATS.Client.Core/NatsConnection.cs b/src/NATS.Client.Core/NatsConnection.cs index 2a0c9005f..f68c80550 100644 --- a/src/NATS.Client.Core/NatsConnection.cs +++ b/src/NATS.Client.Core/NatsConnection.cs @@ -1171,18 +1171,22 @@ private void ThrowIfDisposed() // Drain subs that already removed themselves from the SubscriptionManager // (e.g. JetStream Fetch on natural completion) but still have a user // iterator reading the channel. Their own DrainAsync waits on the - // reader-exited signal. - private async ValueTask DrainRemainingParticipantsAsync() + // reader-exited signal. Run in parallel so a slow/dead server doesn't + // multiply the dispose wait by the number of subs. + private ValueTask DrainRemainingParticipantsAsync() { NatsSubBase[] remaining; lock (_drainParticipantsGate) { if (_drainParticipants.Count == 0) - return; + return default; remaining = _drainParticipants.ToArray(); } - foreach (var sub in remaining) - await sub.DrainAsync().ConfigureAwait(false); + var tasks = new Task[remaining.Length]; + for (var i = 0; i < remaining.Length; i++) + tasks[i] = remaining[i].DrainAsync().AsTask(); + + return new ValueTask(Task.WhenAll(tasks)); } } diff --git a/src/NATS.Client.Core/NatsOpts.cs b/src/NATS.Client.Core/NatsOpts.cs index bfffca9d5..d5c59e9f7 100644 --- a/src/NATS.Client.Core/NatsOpts.cs +++ b/src/NATS.Client.Core/NatsOpts.cs @@ -166,12 +166,13 @@ public sealed record NatsOpts /// /// /// Drain mechanism: each subscription sends UNSUB, then a PING and waits - /// for the PONG (up to 5 seconds). Once the PONG arrives the server has - /// processed UNSUB and the socket reader has delivered any messages the - /// server sent before that, so completing the user channel afterwards - /// won't drop in-flight messages. Dispose therefore blocks for at most - /// one round-trip per drained subscription, plus the - /// wait if set. + /// for the PONG (bounded by ). Once the + /// PONG arrives the server has processed UNSUB and the socket reader has + /// delivered any messages the server sent before that, so completing the + /// user channel afterwards won't drop in-flight messages. Subscriptions + /// drain in parallel on connection dispose, so total dispose time is + /// bounded by one round-trip plus the + /// wait if set, not multiplied by the number of subscriptions. /// /// public bool DrainSubscriptionsOnDispose { get; init; } = false; @@ -189,6 +190,18 @@ public sealed record NatsOpts /// public TimeSpan? ConsumerDrainOnDisposeTimeout { get; init; } = null; + /// + /// Per-subscription PING/PONG timeout used during drain on dispose. + /// (default: 5 seconds) + /// + /// + /// Bounds how long drain will wait for the server to ack the UNSUB before + /// completing the user channel anyway. Increase for high-latency networks + /// where a single round-trip can exceed the default. Only effective when + /// is true. + /// + public TimeSpan DrainPingTimeout { get; init; } = TimeSpan.FromSeconds(5); + /// /// Maximum number of reconnect attempts. (default: -1, unlimited) /// diff --git a/src/NATS.Client.Core/NatsSubBase.cs b/src/NATS.Client.Core/NatsSubBase.cs index 2ff14538b..0276cb23f 100644 --- a/src/NATS.Client.Core/NatsSubBase.cs +++ b/src/NATS.Client.Core/NatsSubBase.cs @@ -402,12 +402,12 @@ internal async ValueTask DrainAsync() // that were in-flight before the UNSUB was received by the server. try { - using var pingCts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + using var pingCts = new CancellationTokenSource(Connection.Opts.DrainPingTimeout); await Connection.PingAsync(pingCts.Token).ConfigureAwait(false); } catch (OperationCanceledException) { - // 5s timeout reached or connection cancelled mid-dispose. Expected on dispose path. + // Timeout reached or connection cancelled mid-dispose. Expected on dispose path. } // Now it's safe to complete the channel, no more messages will arrive. diff --git a/tools/site_src/blog/2026-04-27-drain-on-dispose.md b/tools/site_src/blog/2026-04-27-drain-on-dispose.md index 090f6906c..d45543368 100644 --- a/tools/site_src/blog/2026-04-27-drain-on-dispose.md +++ b/tools/site_src/blog/2026-04-27-drain-on-dispose.md @@ -22,6 +22,9 @@ There are now two opt-in options on `NatsOpts` to control this: consumer and fetch dispose waits up to this timeout for the user's consume loop to finish acking the buffered messages before returning. Only effective when `DrainSubscriptionsOnDispose` is also `true`. +- `DrainPingTimeout` (default `5s`). Per-subscription PING/PONG timeout + used during drain. Increase for high-latency networks where a single + round-trip can exceed the default. ```csharp var opts = NatsOpts.Default with From e17593067cdab24be8022461e10e04c2784ded31 Mon Sep 17 00:00:00 2001 From: Ziya Suzen Date: Wed, 29 Apr 2026 13:04:19 +0100 Subject: [PATCH 17/19] drain: fix routing cleared before PING/PONG, parallelise manager dispose --- .../Internal/SubscriptionManager.cs | 49 ++++++++++++++++--- src/NATS.Client.Core/NatsSubBase.cs | 25 ++++++++-- 2 files changed, 62 insertions(+), 12 deletions(-) diff --git a/src/NATS.Client.Core/Internal/SubscriptionManager.cs b/src/NATS.Client.Core/Internal/SubscriptionManager.cs index ce0ee5bcb..928d80acc 100644 --- a/src/NATS.Client.Core/Internal/SubscriptionManager.cs +++ b/src/NATS.Client.Core/Internal/SubscriptionManager.cs @@ -149,18 +149,23 @@ public async ValueTask DisposeAsync() _cts.Cancel(); #endif - WeakReference[] subRefs; + var subs = new List(); lock (_gate) { - subRefs = _bySid.Values.Select(m => m.WeakReference).ToArray(); - _bySid.Clear(); + foreach (var sidMetadata in _bySid.Values) + { + if (sidMetadata.WeakReference.TryGetTarget(out var sub)) + subs.Add(sub); + } } - foreach (var subRef in subRefs) - { - if (subRef.TryGetTarget(out var sub)) - await sub.DisposeAsync().ConfigureAwait(false); - } + if (subs.Count == 0) + return; + + var tasks = new Task[subs.Count]; + for (var i = 0; i < subs.Count; i++) + tasks[i] = subs[i].DisposeAsync().AsTask(); + await Task.WhenAll(tasks).ConfigureAwait(false); } public ValueTask RemoveAsync(NatsSubBase sub) @@ -188,6 +193,34 @@ public ValueTask RemoveAsync(NatsSubBase sub) return _connection.UnsubscribeAsync(subMetadata.Sid); } + // Send UNSUB and remove from _bySub, but keep in _bySid until after PING/PONG. + // Returns the SID so the caller can call RemoveFromRouting after the round-trip. + internal async ValueTask UnsubscribeForDrainAsync(NatsSubBase sub) + { + SubscriptionMetadata? subMetadata; + lock (_gate) + { + if (!_bySub.TryGetValue(sub, out subMetadata)) + return -1; + _bySub.Remove(sub); + } + + if (_debug) + _logger.LogDebug(NatsLogEvents.Subscription, "Drain-unsubscribing {Subject}/{Sid}", sub.Subject, subMetadata.Sid); + + await _connection.UnsubscribeAsync(subMetadata.Sid).ConfigureAwait(false); + return subMetadata.Sid; + } + + // Remove from _bySid after the PING/PONG round-trip completes. + internal void RemoveFromRouting(int sid) + { + lock (_gate) + { + _bySid.TryRemove(sid, out _); + } + } + /// /// Returns commands for all the live subscriptions to be used on reconnect so that they can rebuild their connection state on the server. /// diff --git a/src/NATS.Client.Core/NatsSubBase.cs b/src/NATS.Client.Core/NatsSubBase.cs index 0276cb23f..b1c0ad5df 100644 --- a/src/NATS.Client.Core/NatsSubBase.cs +++ b/src/NATS.Client.Core/NatsSubBase.cs @@ -393,9 +393,16 @@ internal async ValueTask DrainAsync() _idleTimeoutTimer?.Change(Timeout.Infinite, Timeout.Infinite); _startUpTimeoutTimer?.Change(Timeout.Infinite, Timeout.Infinite); - // Send UNSUB to the server and remove from subscription manager. - // This stops the server from sending new messages for this subscription. - await _manager.RemoveAsync(this).ConfigureAwait(false); + // For the built-in SubscriptionManager: send UNSUB but keep the SID in the routing + // table until after the PING/PONG round-trip, so messages already in the socket + // buffer are delivered before the channel is completed. + // For external managers: fall back to RemoveAsync (routing removed before PING). + var mgr = _manager as SubscriptionManager; + var sid = mgr != null + ? await mgr.UnsubscribeForDrainAsync(this).ConfigureAwait(false) + : -1; + if (mgr == null) + await _manager.RemoveAsync(this).ConfigureAwait(false); // PING/PONG round-trip: after we get the PONG back, we know the server // has processed our UNSUB and the socket reader has processed all messages @@ -407,8 +414,15 @@ internal async ValueTask DrainAsync() } catch (OperationCanceledException) { - // Timeout reached or connection cancelled mid-dispose. Expected on dispose path. + // Timeout via DrainPingTimeout. Expected on dispose path. } + catch (NatsException) + { + // Connection failed or disconnected during drain. Drain is best-effort. + } + + // Remove from routing now that no more messages can arrive for this SID. + mgr?.RemoveFromRouting(sid); // Now it's safe to complete the channel, no more messages will arrive. TryComplete(); @@ -446,6 +460,9 @@ internal async ValueTask DrainAsync() /// internal void MarkReaderActive() { + if (!Connection.Opts.DrainSubscriptionsOnDispose) + return; + if (_readerExited is null) { var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); From daeb7c5949e1ae627e637ed2f2145baa225206d7 Mon Sep 17 00:00:00 2001 From: Ziya Suzen Date: Wed, 29 Apr 2026 13:37:52 +0100 Subject: [PATCH 18/19] drain: guard sid sentinel and ensure socket cleanup on sub-dispose throw --- src/NATS.Client.Core/NatsConnection.cs | 14 ++++++++++---- src/NATS.Client.Core/NatsSubBase.cs | 3 ++- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/NATS.Client.Core/NatsConnection.cs b/src/NATS.Client.Core/NatsConnection.cs index f68c80550..ae2aaad2e 100644 --- a/src/NATS.Client.Core/NatsConnection.cs +++ b/src/NATS.Client.Core/NatsConnection.cs @@ -295,10 +295,16 @@ public virtual async ValueTask DisposeAsync() { // Drain subs and flush the writer first so UNSUB/PING/PONG // and any pending acks can land before the socket closes. - await _subscriptionManager.DisposeAsync().ConfigureAwait(false); - await DrainRemainingParticipantsAsync().ConfigureAwait(false); - await CommandWriter.DisposeAsync().ConfigureAwait(false); - await DisposeSocketAsync(false).ConfigureAwait(false); + try + { + await _subscriptionManager.DisposeAsync().ConfigureAwait(false); + } + finally + { + await DrainRemainingParticipantsAsync().ConfigureAwait(false); + await CommandWriter.DisposeAsync().ConfigureAwait(false); + await DisposeSocketAsync(false).ConfigureAwait(false); + } } else { diff --git a/src/NATS.Client.Core/NatsSubBase.cs b/src/NATS.Client.Core/NatsSubBase.cs index b1c0ad5df..afa3feb70 100644 --- a/src/NATS.Client.Core/NatsSubBase.cs +++ b/src/NATS.Client.Core/NatsSubBase.cs @@ -422,7 +422,8 @@ internal async ValueTask DrainAsync() } // Remove from routing now that no more messages can arrive for this SID. - mgr?.RemoveFromRouting(sid); + if (sid != -1) + mgr?.RemoveFromRouting(sid); // Now it's safe to complete the channel, no more messages will arrive. TryComplete(); From fd26de89b5f7043ea91d901b9a14d19de1224aec Mon Sep 17 00:00:00 2001 From: Ziya Suzen Date: Wed, 29 Apr 2026 13:56:33 +0100 Subject: [PATCH 19/19] js: remove dead _disposed field from consumer sub classes --- src/NATS.Client.JetStream/Internal/NatsJSConsume.cs | 2 -- src/NATS.Client.JetStream/Internal/NatsJSFetch.cs | 2 -- src/NATS.Client.JetStream/Internal/NatsJSOrderedConsume.cs | 3 --- 3 files changed, 7 deletions(-) diff --git a/src/NATS.Client.JetStream/Internal/NatsJSConsume.cs b/src/NATS.Client.JetStream/Internal/NatsJSConsume.cs index 0067aba4d..09e295c36 100644 --- a/src/NATS.Client.JetStream/Internal/NatsJSConsume.cs +++ b/src/NATS.Client.JetStream/Internal/NatsJSConsume.cs @@ -45,7 +45,6 @@ internal class NatsJSConsume : NatsSubBase private readonly object _pendingGate = new(); private long _pendingMsgs; private long _pendingBytes; - private int _disposed; private int _consecutive503Errors; public NatsJSConsume( @@ -217,7 +216,6 @@ public void Delivered(int msgSize) public override async ValueTask DisposeAsync() { - Interlocked.Exchange(ref _disposed, 1); try { await DrainAsync().ConfigureAwait(false); diff --git a/src/NATS.Client.JetStream/Internal/NatsJSFetch.cs b/src/NATS.Client.JetStream/Internal/NatsJSFetch.cs index 26ecd16da..5dcf6d429 100644 --- a/src/NATS.Client.JetStream/Internal/NatsJSFetch.cs +++ b/src/NATS.Client.JetStream/Internal/NatsJSFetch.cs @@ -32,7 +32,6 @@ internal class NatsJSFetch : NatsSubBase private long _pendingMsgs; private long _pendingBytes; - private int _disposed; public NatsJSFetch( long maxMsgs, @@ -162,7 +161,6 @@ public void ResetHeartbeatTimer() public override async ValueTask DisposeAsync() { - Interlocked.Exchange(ref _disposed, 1); try { await DrainAsync().ConfigureAwait(false); diff --git a/src/NATS.Client.JetStream/Internal/NatsJSOrderedConsume.cs b/src/NATS.Client.JetStream/Internal/NatsJSOrderedConsume.cs index 78d61cd97..900275d85 100644 --- a/src/NATS.Client.JetStream/Internal/NatsJSOrderedConsume.cs +++ b/src/NATS.Client.JetStream/Internal/NatsJSOrderedConsume.cs @@ -34,7 +34,6 @@ internal class NatsJSOrderedConsume : NatsSubBase private readonly object _pendingGate = new(); private long _pendingMsgs; private long _pendingBytes; - private int _disposed; public NatsJSOrderedConsume( long maxMsgs, @@ -134,8 +133,6 @@ public ValueTask CallMsgNextAsync(string origin, ConsumerGetnextRequest request, public override async ValueTask DisposeAsync() { - Interlocked.Exchange(ref _disposed, 1); - _context.Connection.ConnectionDisconnected -= ConnectionOnConnectionDisconnected; try {