From 1b2569689b590af32b2a48209cd3d79683f7e9e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Guimar=C3=A3es?= Date: Sun, 4 Oct 2026 19:09:47 +0100 Subject: [PATCH] fix(sync): back off after failed syncs; handle IBKR 1025 lockout; cap failed Flex probes --- .../src/Host.Api/Jobs/BrokerSyncService.cs | 28 +++++------ .../Sync/SyncSchedule.cs | 44 +++++++++++++++++ .../Ibkr/FlexClient.cs | 5 ++ .../Ibkr/IbkrFlexProvider.cs | 14 ++++++ .../tests/Integrations.Tests/IbkrFlexTests.cs | 18 ++++++- .../Integrations.Tests/SyncScheduleTests.cs | 48 +++++++++++++++++++ web/public/i18n/en.json | 3 +- web/public/i18n/pt-PT.json | 3 +- .../features/connections/credentials.spec.ts | 4 ++ .../app/features/connections/credentials.ts | 1 + 10 files changed, 151 insertions(+), 17 deletions(-) create mode 100644 backend/src/Modules/Integrations/Integrations.Application/Sync/SyncSchedule.cs create mode 100644 backend/tests/Integrations.Tests/SyncScheduleTests.cs diff --git a/backend/src/Host.Api/Jobs/BrokerSyncService.cs b/backend/src/Host.Api/Jobs/BrokerSyncService.cs index d0ab5e9..def4fe0 100644 --- a/backend/src/Host.Api/Jobs/BrokerSyncService.cs +++ b/backend/src/Host.Api/Jobs/BrokerSyncService.cs @@ -8,7 +8,8 @@ namespace Host.Api.Jobs; /// /// Runs broker syncs one at a time: manual requests from the queue, plus a schedule — Trading 212 every 4 hours, -/// IBKR once a day after its end-of-day processing. Connections needing attention wait for the user. +/// IBKR once a day after its end-of-day processing. Connections needing attention wait for the user; failed +/// attempts back off (see ). /// internal sealed class BrokerSyncService(IServiceScopeFactory scopes, SyncQueue queue, TimeProvider clock, ILogger logger) : BackgroundService @@ -38,8 +39,19 @@ private async Task ScheduleAsync(CancellationToken ct) var db = scope.ServiceProvider.GetRequiredService(); var active = await db.Connections.AsNoTracking() .Where(c => c.Status == ConnectionStatus.Active).ToListAsync(ct); + var ids = active.Select(c => c.Id).ToList(); var now = clock.GetUtcNow(); - due = active.Where(c => IsDue(c, now)).ToList(); + // Only recent jobs matter for the back-off (the longest wait is 12 h). + var since = now.AddDays(-2); + var lastJobs = (await db.SyncJobs.AsNoTracking() + .Where(j => ids.Contains(j.ConnectionId) && j.FinishedAtUtc != null && j.StartedAtUtc >= since) + .ToListAsync(ct)) + .GroupBy(j => j.ConnectionId) + .Select(g => g.MaxBy(j => j.StartedAtUtc)!) + .ToDictionary(j => j.ConnectionId, j => new LastAttempt(j.StartedAtUtc, + j.Outcome == SyncOutcome.Failed, SyncSchedule.IsLockout(j.Errors))); + due = active.Where(c => SyncSchedule.IsDue(c.Kind, c.LastSuccessfulSyncUtc, + lastJobs.GetValueOrDefault(c.Id), now)).ToList(); } foreach (var connection in due) @@ -49,18 +61,6 @@ private async Task ScheduleAsync(CancellationToken ct) } while (await timer.WaitForNextTickAsync(ct)); } - private static bool IsDue(BrokerConnection c, DateTimeOffset now) - { - var last = c.LastSuccessfulSyncUtc ?? DateTimeOffset.MinValue; - return c.Kind switch - { - BrokerKind.Trading212 => now - last > TimeSpan.FromHours(4), - // Flex data is end-of-day: one run per day, after 06:00 UTC. - BrokerKind.InteractiveBrokers => now.Hour >= 6 && last.UtcDateTime.Date < now.UtcDateTime.Date, - _ => now - last > TimeSpan.FromHours(24), - }; - } - private async Task RunAsync(Guid connectionId, SyncTrigger trigger, CancellationToken ct) { await _oneAtATime.WaitAsync(ct); diff --git a/backend/src/Modules/Integrations/Integrations.Application/Sync/SyncSchedule.cs b/backend/src/Modules/Integrations/Integrations.Application/Sync/SyncSchedule.cs new file mode 100644 index 0000000..ec21577 --- /dev/null +++ b/backend/src/Modules/Integrations/Integrations.Application/Sync/SyncSchedule.cs @@ -0,0 +1,44 @@ +using Integrations.Application.Contracts; + +namespace Integrations.Application.Sync; + +/// The last scheduled-or-manual attempt for a connection, as far as the schedule cares. +public sealed record LastAttempt(DateTimeOffset StartedAtUtc, bool Failed, bool LockedOut); + +/// +/// When a connection is due for a scheduled sync. A failed attempt backs off instead of retrying on every tick: +/// brokers count failed requests, and IBKR blocks the token after too many (Flex error 1025). +/// +public static class SyncSchedule +{ + public static readonly TimeSpan AfterLockout = TimeSpan.FromHours(12); + + public static TimeSpan AfterFailure(BrokerKind kind) => + kind == BrokerKind.InteractiveBrokers ? TimeSpan.FromHours(6) : TimeSpan.FromHours(2); + + public static bool IsDue(BrokerKind kind, DateTimeOffset? lastSuccess, LastAttempt? last, DateTimeOffset now) + { + if (last is { Failed: true }) + { + var wait = last.LockedOut ? AfterLockout : AfterFailure(kind); + if (now - last.StartedAtUtc < wait) + { + return false; + } + } + + var success = lastSuccess ?? DateTimeOffset.MinValue; + return kind switch + { + BrokerKind.Trading212 => now - success > TimeSpan.FromHours(4), + // Flex data is end-of-day: one run per day, after 06:00 UTC. + BrokerKind.InteractiveBrokers => now.Hour >= 6 && success.UtcDateTime.Date < now.UtcDateTime.Date, + _ => now - success > TimeSpan.FromHours(24), + }; + } + + /// Recognises a broker-side lockout in a stored job error. + public static bool IsLockout(string? errors) => + errors is not null && (errors.Contains("1025", StringComparison.Ordinal) || + errors.Contains("too many failed attempts", StringComparison.OrdinalIgnoreCase)); +} diff --git a/backend/src/Modules/Integrations/Integrations.Infrastructure/Ibkr/FlexClient.cs b/backend/src/Modules/Integrations/Integrations.Infrastructure/Ibkr/FlexClient.cs index 34c23e3..1857532 100644 --- a/backend/src/Modules/Integrations/Integrations.Infrastructure/Ibkr/FlexClient.cs +++ b/backend/src/Modules/Integrations/Integrations.Infrastructure/Ibkr/FlexClient.cs @@ -76,6 +76,11 @@ private static Exception Error(FlexStatus? status) // "Statement is not available": usually a date range before the account existed. The provider narrows // the range; only when nothing at all is available does it become a configuration problem. 1003 => new FlexStatementUnavailableException(message), + // Undocumented by IBKR: the token is blocked for a while after too many failed requests. Scheduled syncs + // back off (SyncSchedule); hammering it only extends the block. + 1025 => new ProviderUnavailableException( + "IBKR temporarily blocked the Flex token after too many failed attempts (Flex error 1025). " + + "Automatic syncs pause for 12 hours; try again later or generate a new Flex token."), 1012 => new ProviderConfigurationException("The IBKR Flex token has expired. Generate a new one in Client Portal."), 1013 => new ProviderConfigurationException("The IBKR Flex token is restricted to another IP address."), 1014 => new ProviderConfigurationException("The IBKR Flex query id is invalid."), diff --git a/backend/src/Modules/Integrations/Integrations.Infrastructure/Ibkr/IbkrFlexProvider.cs b/backend/src/Modules/Integrations/Integrations.Infrastructure/Ibkr/IbkrFlexProvider.cs index ddd7a2f..600b623 100644 --- a/backend/src/Modules/Integrations/Integrations.Infrastructure/Ibkr/IbkrFlexProvider.cs +++ b/backend/src/Modules/Integrations/Integrations.Infrastructure/Ibkr/IbkrFlexProvider.cs @@ -182,6 +182,7 @@ private async Task> BackfillAsync(DateOnly start, DateOnly e var statements = new List(); var unavailable = false; + _failuresLeft = MaxFailedRequests; foreach (var (from, windowEnd) in Enumerable.Reverse(windows)) { var to = windowEnd; @@ -221,8 +222,20 @@ private async Task> BackfillAsync(DateOnly start, DateOnly e return statements; } + /// + /// IBKR blocks a token after too many failed requests (1025), so one sync never spends more than a few on + /// probing date ranges; the rest of the history is picked up by later syncs. + /// + private const int MaxFailedRequests = 4; + private int _failuresLeft = MaxFailedRequests; + private async Task TryFetchAsync(DateOnly from, DateOnly to, List into, CancellationToken ct) { + if (_failuresLeft <= 0) + { + return false; + } + try { into.AddRange(Statements(await client.FetchAsync(token, queryId, from, to, ct))); @@ -230,6 +243,7 @@ private async Task TryFetchAsync(DateOnly from, DateOnly to, ListFail1025Too many failed attempts"); + + var error = await Should.ThrowAsync( + () => Provider(http).GetTransactionsAsync(new DateTimeOffset(2026, 9, 15, 0, 0, 0, TimeSpan.Zero), CancellationToken.None)); + + error.Message.ShouldContain("1025"); + error.Message.ShouldContain("12 hours"); + http.Requests.Count.ShouldBe(1); + } + [Fact] public async Task When_no_period_has_a_statement_the_error_explains_what_to_check() { @@ -150,7 +164,9 @@ public async Task When_no_period_has_a_statement_the_error_explains_what_to_chec error.Message.ShouldContain("1003"); error.Message.ShouldContain("Query ID"); error.Message.ShouldContain("Last 365 Calendar Days"); - http.Requests.Count.ShouldBeLessThan(16); // a few earlier end dates, then halving: it gives up quickly + // IBKR blocks a token after too many failed requests (1025): one sync spends at most four on probing. + http.Requests.Count(r => r.RequestUri!.AbsolutePath.EndsWith("SendRequest", StringComparison.Ordinal)) + .ShouldBeLessThanOrEqualTo(4); } [Fact] diff --git a/backend/tests/Integrations.Tests/SyncScheduleTests.cs b/backend/tests/Integrations.Tests/SyncScheduleTests.cs new file mode 100644 index 0000000..f08140d --- /dev/null +++ b/backend/tests/Integrations.Tests/SyncScheduleTests.cs @@ -0,0 +1,48 @@ +using Integrations.Application.Contracts; +using Integrations.Application.Sync; + +namespace Integrations.Tests; + +public sealed class SyncScheduleTests +{ + private static readonly DateTimeOffset Now = new(2026, 10, 5, 12, 0, 0, TimeSpan.Zero); + + [Fact] + public void A_failed_ibkr_sync_is_not_retried_on_every_tick() + { + var yesterday = Now.AddDays(-1); + var failedRecently = new LastAttempt(Now.AddMinutes(-30), Failed: true, LockedOut: false); + var failedLongAgo = new LastAttempt(Now.AddHours(-7), Failed: true, LockedOut: false); + + SyncSchedule.IsDue(BrokerKind.InteractiveBrokers, yesterday, failedRecently, Now).ShouldBeFalse(); + SyncSchedule.IsDue(BrokerKind.InteractiveBrokers, yesterday, failedLongAgo, Now).ShouldBeTrue(); + } + + [Fact] + public void A_lockout_pauses_scheduled_syncs_for_twelve_hours() + { + var locked = new LastAttempt(Now.AddHours(-8), Failed: true, LockedOut: true); + + SyncSchedule.IsDue(BrokerKind.InteractiveBrokers, Now.AddDays(-2), locked, Now).ShouldBeFalse(); + SyncSchedule.IsDue(BrokerKind.InteractiveBrokers, Now.AddDays(-2), locked, Now.AddHours(5)).ShouldBeTrue(); + } + + [Fact] + public void Successful_syncs_keep_the_normal_cadence() + { + var ok = new LastAttempt(Now.AddHours(-5), Failed: false, LockedOut: false); + + SyncSchedule.IsDue(BrokerKind.Trading212, Now.AddHours(-5), ok, Now).ShouldBeTrue(); + SyncSchedule.IsDue(BrokerKind.Trading212, Now.AddHours(-1), ok, Now).ShouldBeFalse(); + SyncSchedule.IsDue(BrokerKind.InteractiveBrokers, Now, ok, Now).ShouldBeFalse(); // already synced today + SyncSchedule.IsDue(BrokerKind.InteractiveBrokers, null, null, Now).ShouldBeTrue(); // never synced + } + + [Theory] + [InlineData("[\"IBKR Flex error 1025: Too many failed attempts\"]", true)] + [InlineData("[\"Too many failed attempts\"]", true)] + [InlineData("[\"IBKR Flex busy (1019)\"]", false)] + [InlineData(null, false)] + public void Recognises_a_lockout_in_stored_errors(string? errors, bool expected) => + SyncSchedule.IsLockout(errors).ShouldBe(expected); +} diff --git a/web/public/i18n/en.json b/web/public/i18n/en.json index ab1f0c9..724051f 100644 --- a/web/public/i18n/en.json +++ b/web/public/i18n/en.json @@ -653,7 +653,8 @@ "ibkrIp": "The Flex token is restricted to a different IP address. Update the IP in Flex Web Service Configuration or generate a token without IP restriction.", "ibkrQueryId": "IBKR does not recognise the query ID. Use the number shown next to your Activity Flex Query.", "ibkrToken": "IBKR does not recognise the token. Copy it again from Flex Web Service Configuration (a new token invalidates the old one).", - "ibkrNoStatement": "IBKR has no statement for this account yet (error 1003). Check that the Query ID is the number of your Activity Flex Query, with Period \"Last 365 Calendar Days\", and that the account already has activity." + "ibkrNoStatement": "IBKR has no statement for this account yet (error 1003). Check that the Query ID is the number of your Activity Flex Query, with Period \"Last 365 Calendar Days\", and that the account already has activity.", + "ibkrLocked": "IBKR temporarily blocked the token after too many failed attempts (error 1025). Automatic syncs pause for 12 hours — don't press Sync meanwhile, it extends the block. If it persists, generate a new Flex token and update the connection." }, "guide": { "title": "How to connect", diff --git a/web/public/i18n/pt-PT.json b/web/public/i18n/pt-PT.json index 2e699bd..b71169a 100644 --- a/web/public/i18n/pt-PT.json +++ b/web/public/i18n/pt-PT.json @@ -653,7 +653,8 @@ "ibkrIp": "O token Flex está restrito a outro endereço IP. Atualiza o IP na Flex Web Service Configuration ou gera um token sem restrição de IP.", "ibkrQueryId": "O IBKR não reconhece o ID da query. Usa o número mostrado junto à tua Activity Flex Query.", "ibkrToken": "O IBKR não reconhece o token. Copia-o de novo da Flex Web Service Configuration (um novo token invalida o anterior).", - "ibkrNoStatement": "O IBKR ainda não tem extrato para esta conta (erro 1003). Confirma que o ID da query é o número da tua Activity Flex Query, com período \"Last 365 Calendar Days\", e que a conta já tem movimentos." + "ibkrNoStatement": "O IBKR ainda não tem extrato para esta conta (erro 1003). Confirma que o ID da query é o número da tua Activity Flex Query, com período \"Last 365 Calendar Days\", e que a conta já tem movimentos.", + "ibkrLocked": "O IBKR bloqueou temporariamente o token depois de demasiadas tentativas falhadas (erro 1025). As sincronizações automáticas param 12 horas — não carregues em Sincronizar entretanto, prolonga o bloqueio. Se persistir, gera um novo token Flex e atualiza a ligação." }, "guide": { "title": "Como ligar", diff --git a/web/src/app/features/connections/credentials.spec.ts b/web/src/app/features/connections/credentials.spec.ts index e7d7f9d..3feb6d2 100644 --- a/web/src/app/features/connections/credentials.spec.ts +++ b/web/src/app/features/connections/credentials.spec.ts @@ -55,6 +55,10 @@ describe('friendlyError', () => { ['The IBKR Flex token is restricted to another IP address.', 'ibkrIp'], ['The IBKR Flex query id is invalid.', 'ibkrQueryId'], ['The IBKR Flex token is invalid.', 'ibkrToken'], + [ + 'IBKR temporarily blocked the Flex token after too many failed attempts (Flex error 1025). Automatic syncs pause for 12 hours; try again later or generate a new Flex token.', + 'ibkrLocked', + ], [ 'IBKR has no statement for the requested period (Flex error 1003). Check that the Query ID belongs to an Activity Flex Query of this account and that the account already has activity.', 'ibkrNoStatement', diff --git a/web/src/app/features/connections/credentials.ts b/web/src/app/features/connections/credentials.ts index c6a6ecd..910f13f 100644 --- a/web/src/app/features/connections/credentials.ts +++ b/web/src/app/features/connections/credentials.ts @@ -57,6 +57,7 @@ export function friendlyError(message: string | null | undefined): string | null [/Flex query id is invalid/i, 'ibkrQueryId'], [/Flex token is invalid/i, 'ibkrToken'], [/no statement for the requested period/i, 'ibkrNoStatement'], + [/too many failed attempts|Flex error 1025/i, 'ibkrLocked'], ]; const hit = rules.find(([pattern]) => pattern.test(message)); return hit ? `connections.errors.${hit[1]}` : null;