diff --git a/backend/LexBoxApi/GraphQL/CustomTypes/IsLanguageForgeProjectDataLoader.cs b/backend/LexBoxApi/GraphQL/CustomTypes/IsLanguageForgeProjectDataLoader.cs index 74037310b..a86c8440b 100644 --- a/backend/LexBoxApi/GraphQL/CustomTypes/IsLanguageForgeProjectDataLoader.cs +++ b/backend/LexBoxApi/GraphQL/CustomTypes/IsLanguageForgeProjectDataLoader.cs @@ -2,19 +2,28 @@ using LfClassicData; using MongoDB.Driver; using MongoDB.Driver.Linq; +using Polly; +using Polly.CircuitBreaker; +using Polly.Fallback; +using Polly.Registry; namespace LexBoxApi.GraphQL.CustomTypes; public class IsLanguageForgeProjectDataLoader : BatchDataLoader, IIsLanguageForgeProjectDataLoader { + public const string ResiliencePolicyName = "IsLanguageForgeProjectDataLoader"; private readonly SystemDbContext _systemDbContext; + private readonly ResiliencePipeline> _resiliencePipeline; public IsLanguageForgeProjectDataLoader( SystemDbContext systemDbContext, IBatchScheduler batchScheduler, + [FromKeyedServices(ResiliencePolicyName)] + ResiliencePipeline> resiliencePipeline, DataLoaderOptions? options = null) : base(batchScheduler, options) { + _resiliencePipeline = resiliencePipeline; _systemDbContext = systemDbContext; } @@ -22,9 +31,72 @@ protected override async Task> LoadBatchAsync( IReadOnlyList projectCodes, CancellationToken cancellationToken) { - return await MongoExtensions.ToAsyncEnumerable(_systemDbContext.Projects.AsQueryable() - .Select(p => p.ProjectCode) - .Where(projectCode => projectCodes.Contains(projectCode))) - .ToDictionaryAsync(projectCode => projectCode, _ => true, cancellationToken); + return await FaultTolerantLoadBatch(projectCodes, cancellationToken); + } + + private async ValueTask> FaultTolerantLoadBatch( + IReadOnlyList projectCodes, + CancellationToken cancellationToken) + { + ResilienceContext context = ResilienceContextPool.Shared.Get(cancellationToken); + context.Properties.Set(ProjectCodesKey, projectCodes); + try + { + return await _resiliencePipeline.ExecuteAsync( + static async (context, state) => await LoadBatch(state, + context.Properties.GetValue(ProjectCodesKey, []), + context.CancellationToken), + context, + this); + } + finally + { + ResilienceContextPool.Shared.Return(context); + } + } + + private static async Task> LoadBatch(IsLanguageForgeProjectDataLoader loader, + IReadOnlyList list, + CancellationToken token) + { + return await MongoExtensions.ToAsyncEnumerable(loader._systemDbContext.Projects.AsQueryable() + .Select(p => p.ProjectCode) + .Where(projectCode => list.Contains(projectCode))) + .ToDictionaryAsync(projectCode => projectCode, _ => true, token); + } + + + public static readonly ResiliencePropertyKey> ProjectCodesKey = new("project-codes"); + + public static ResiliencePipelineBuilder> ConfigureResiliencePipeline( + ResiliencePipelineBuilder> builder, TimeSpan circuitBreakerDuration) + { + var circuitBreakerStrategyOptions = new CircuitBreakerStrategyOptions> + { + //docs https://www.pollydocs.org/strategies/circuit-breaker.html + Name = "IsLanguageForgeProjectDataLoaderCircuitBreaker", + MinimumThroughput = 2,//must be at least 2 + BreakDuration = circuitBreakerDuration, + //window in which the minimum throughput can be reached. + //ff there is only 1 failure in an hour, then the circuit will not break, + //but the moment there is a second failure then it will break immediately. + SamplingDuration = TimeSpan.FromHours(1), + }; + var fallbackStrategyOptions = new FallbackStrategyOptions>() + { + //docs https://www.pollydocs.org/strategies/fallback.html + Name = "IsLanguageForgeProjectDataLoaderFallback", + FallbackAction = arguments => + { + IReadOnlyDictionary emptyResult = arguments.Context.Properties + .GetValue(ProjectCodesKey, []).ToDictionary(pc => pc, _ => false); + return Outcome.FromResultAsValueTask(emptyResult); + } + }; + builder + .AddFallback(fallbackStrategyOptions) + .AddCircuitBreaker(circuitBreakerStrategyOptions) + ; + return builder; } } diff --git a/backend/LexBoxApi/LexBoxApi.csproj b/backend/LexBoxApi/LexBoxApi.csproj index d7086c816..b87982f40 100644 --- a/backend/LexBoxApi/LexBoxApi.csproj +++ b/backend/LexBoxApi/LexBoxApi.csproj @@ -48,6 +48,8 @@ + + diff --git a/backend/LexBoxApi/LexBoxKernel.cs b/backend/LexBoxApi/LexBoxKernel.cs index c4dc4580c..7b420421c 100644 --- a/backend/LexBoxApi/LexBoxKernel.cs +++ b/backend/LexBoxApi/LexBoxKernel.cs @@ -7,7 +7,10 @@ using LexCore.Config; using LexCore.ServiceInterfaces; using LexSyncReverseProxy; +using LfClassicData; using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Options; +using Polly; using Swashbuckle.AspNetCore.Swagger; namespace LexBoxApi; @@ -55,6 +58,11 @@ public static void AddLexBoxApi(this IServiceCollection services, services.AddHostedService(); services.AddTransient(); services.AddScoped(); + services.AddResiliencePipeline>(IsLanguageForgeProjectDataLoader.ResiliencePolicyName, (builder, context) => + { + builder.ConfigureTelemetry(context.ServiceProvider.GetRequiredService()); + IsLanguageForgeProjectDataLoader.ConfigureResiliencePipeline(builder, context.ServiceProvider.GetRequiredService>().Value.IsLfProjectConnectionRetryTimeout); + }); services.AddScoped(); services.AddSingleton(); services.AddSingleton(); diff --git a/backend/LexBoxApi/appsettings.Development.json b/backend/LexBoxApi/appsettings.Development.json index 57ec29d7b..fe5d83396 100644 --- a/backend/LexBoxApi/appsettings.Development.json +++ b/backend/LexBoxApi/appsettings.Development.json @@ -13,7 +13,10 @@ "ConnectionString": "mongodb://localhost:27017", "AuthSource": "admin", "Username": "admin", - "Password": "pass" + "Password": "pass", + "ServerSelectionTimeout": "00:00:01", + "ConnectTimeout": "00:00:01", + "IsLfProjectConnectionRetryTimeout": "00:10:00" }, "ForwardedHeadersOptions": { "KnownNetworks": [ diff --git a/backend/LfClassicData/DataServiceKernel.cs b/backend/LfClassicData/DataServiceKernel.cs index 39280bc10..edba8a807 100644 --- a/backend/LfClassicData/DataServiceKernel.cs +++ b/backend/LfClassicData/DataServiceKernel.cs @@ -21,24 +21,25 @@ public static void AddLanguageForgeClassicMiniLcm(this IServiceCollection servic services.AddSingleton(BuildMongoClientSettings); services.AddSingleton(provider => new MongoClient(provider.GetRequiredService())); services.AddSingleton(); - services.AddSingleton(); services.AddSingleton(); } public static MongoClientSettings BuildMongoClientSettings(IServiceProvider provider) { - var config = provider.GetRequiredService>(); - var mongoSettings = MongoClientSettings.FromConnectionString(config.Value.ConnectionString); - if (config.Value.HasCredentials) + var config = provider.GetRequiredService>().Value; + var mongoSettings = MongoClientSettings.FromConnectionString(config.ConnectionString); + if (config.HasCredentials) { mongoSettings.Credential = MongoCredential.CreateCredential( - databaseName: config.Value.AuthSource, - username: config.Value.Username, - password: config.Value.Password + databaseName: config.AuthSource, + username: config.Username, + password: config.Password ); } mongoSettings.LoggingSettings = new LoggingSettings(provider.GetRequiredService()); + mongoSettings.ConnectTimeout = config.ConnectTimeout; + mongoSettings.ServerSelectionTimeout = config.ServerSelectionTimeout; mongoSettings.ClusterConfigurator = cb => cb.Subscribe(new DiagnosticsActivityEventSubscriber(new() { CaptureCommandText = true })); return mongoSettings; diff --git a/backend/LfClassicData/LfClassicConfig.cs b/backend/LfClassicData/LfClassicConfig.cs index 2634f60db..ee99301e3 100644 --- a/backend/LfClassicData/LfClassicConfig.cs +++ b/backend/LfClassicData/LfClassicConfig.cs @@ -10,5 +10,11 @@ public class LfClassicConfig public string? AuthSource { get; set; } public string? Username { get; set; } public string? Password { get; set; } + public TimeSpan ConnectTimeout { get; set; } = TimeSpan.FromSeconds(5); + public TimeSpan ServerSelectionTimeout { get; set; } = TimeSpan.FromSeconds(5); + /// + /// how long to wait before trying to determine if a project is an LF project after a failure + /// + public TimeSpan IsLfProjectConnectionRetryTimeout { get; set; } = TimeSpan.FromSeconds(60); public bool HasCredentials => AuthSource is not null && Username is not null && Password is not null; } diff --git a/backend/Testing/Services/IsLanguageForgeProjectDataLoaderTests.cs b/backend/Testing/Services/IsLanguageForgeProjectDataLoaderTests.cs new file mode 100644 index 000000000..8f48a0929 --- /dev/null +++ b/backend/Testing/Services/IsLanguageForgeProjectDataLoaderTests.cs @@ -0,0 +1,89 @@ +using LexBoxApi.GraphQL.CustomTypes; +using Microsoft.Extensions.Time.Testing; +using Polly; +using Shouldly; + +namespace Testing.Services; + +public class IsLanguageForgeProjectDataLoaderTests +{ + private readonly FakeTimeProvider _timeProvider = new(); + private readonly ResiliencePipeline> _pipeline; + private static readonly TimeSpan BreakDuration = TimeSpan.FromSeconds(60); + + public IsLanguageForgeProjectDataLoaderTests() + { + _pipeline = IsLanguageForgeProjectDataLoader.ConfigureResiliencePipeline(new() { TimeProvider = _timeProvider }, BreakDuration) + .Build(); + } + + private ValueTask>> Execute(Exception? exception = null) + { + ResilienceContext context = ResilienceContextPool.Shared.Get(); + context.Properties.Set(IsLanguageForgeProjectDataLoader.ProjectCodesKey, new[] { "test" }); + return _pipeline.ExecuteOutcomeAsync((context, state) => + { + if (exception is not null) + { + return Outcome.FromExceptionAsValueTask>(exception); + } + + return Outcome.FromResultAsValueTask(new Dictionary() { { "test", true } }); + }, + context, + this); + } + + private void VerifyEmptyResult(Outcome> result) + { + result.Exception.ShouldBeNull(); + result.Result.ShouldBe(new Dictionary() { { "test", false } }); + } + + private void VerifySuccessResult(Outcome> result) + { + result.Exception.ShouldBeNull(); + result.Result.ShouldBe(new Dictionary() { { "test", true } }); + } + + [Fact] + public async Task ResiliencePipelineWorksFine() + { + var result = await Execute(); + VerifySuccessResult(result); + } + + [Fact] + public async Task ResiliencePipelineReturnsEmptyResultWhenExceptionIsThrown() + { + var result = await Execute(new Exception("test")); + VerifyEmptyResult(result); + } + + [Fact] + public async Task CircuitBreaksAfter2Failures() + { + for (int i = 0; i < 3; i++) + { + await Execute(new Exception("test")); + _timeProvider.Advance(TimeSpan.FromSeconds(21)); + } + //the circuit is open, now the fallback should be used + var result = await Execute(); + VerifyEmptyResult(result); + } + + [Fact] + public async Task CircuitBreaksAndReOpensAfterTimeout() + { + for (int i = 0; i < 3; i++) + { + await Execute(new Exception("test")); + _timeProvider.Advance(TimeSpan.FromSeconds(21)); + } + //the circuit is open, now the fallback should be used + VerifyEmptyResult(await Execute()); + _timeProvider.Advance(BreakDuration + TimeSpan.FromSeconds(1)); + VerifySuccessResult(await Execute()); + } +} diff --git a/backend/Testing/Testing.csproj b/backend/Testing/Testing.csproj index 556f03705..1ff79f29b 100644 --- a/backend/Testing/Testing.csproj +++ b/backend/Testing/Testing.csproj @@ -28,6 +28,7 @@ +