feat(api): Redis distributed cache + durable background job queue (#217 #218)

src/Orbit.Api/Extensions/ServiceCollectionExtensions.cs

@@ -1,5 +1,7 @@
 using System.Text;
 using FluentValidation;
+using Hangfire;
+using Hangfire.PostgreSql;
 using Microsoft.AspNetCore.Authentication;
 using Microsoft.AspNetCore.Authentication.JwtBearer;
 using Microsoft.EntityFrameworkCore;
@@ -23,6 +25,7 @@
 using Orbit.Application.Habits.Validators;
 using Orbit.Domain.Interfaces;
 using Orbit.Infrastructure.AI;
+using Orbit.Infrastructure.BackgroundJobs;
 using Orbit.Infrastructure.Configuration;
 using Orbit.Infrastructure.Persistence;
 using Orbit.Infrastructure.Services;
@@ -341,6 +344,7 @@ public static WebApplicationBuilder AddOrbitInfrastructure(this WebApplicationBu
             .ConfigureHttpClient(c => c.Timeout = httpTimeout);
 
         builder.Services.AddMemoryCache();
+        AddOrbitDistributedCache(builder);
 
         builder.Services.AddValidatorsFromAssemblyContaining<CreateHabitCommandValidator>();
 
@@ -444,6 +448,24 @@ private static void AddPushAndReferralServices(WebApplicationBuilder builder, Ti
     }
 
     private static void AddBackgroundServices(WebApplicationBuilder builder)
+    {
+        builder.Services.AddScoped<ISlipAlertMessageService, AiSlipAlertMessageService>();
+
+        var useDurableQueue = builder.Configuration.GetSection(BackgroundJobSettings.SectionName)
+            .Get<BackgroundJobSettings>()?.UseDurableQueue ?? false;
+
+        builder.Services.AddHostedService<DataEncryptionMigrationService>();
+
+        if (useDurableQueue)
+            AddDurableRecurringJobs(builder);
+        else
+            AddInProcessSchedulers(builder);
+
+        builder.Services.AddHealthChecks()
+            .AddCheck<BackgroundServiceHealthCheck>("background-services");
+    }
+
+    private static void AddInProcessSchedulers(WebApplicationBuilder builder)
     {
         builder.Services.AddHostedService<ReminderSchedulerService>();
         builder.Services.AddHostedService<GoalDeadlineNotificationService>();
@@ -452,14 +474,67 @@ private static void AddBackgroundServices(WebApplicationBuilder builder)
         builder.Services.AddHostedService<HabitDueDateAdvancementService>();
         builder.Services.AddHostedService<StreakGoalSyncService>();
         builder.Services.AddHostedService<StreakFreezeAutoActivationService>();
-        builder.Services.AddHostedService<DataEncryptionMigrationService>();
         builder.Services.AddHostedService<SyncCleanupService>();
         builder.Services.AddHostedService<PlayNotificationCleanupService>();
         builder.Services.AddHostedService<CalendarAutoSyncService>();
-        builder.Services.AddScoped<ISlipAlertMessageService, AiSlipAlertMessageService>();
+    }
 
-        builder.Services.AddHealthChecks()
-            .AddCheck<BackgroundServiceHealthCheck>("background-services");
+    private static void AddDurableRecurringJobs(WebApplicationBuilder builder)
+    {
+        var connectionString = builder.Configuration.GetConnectionString("DefaultConnection");
+        if (string.IsNullOrWhiteSpace(connectionString))
+            throw new InvalidOperationException(
+                $"{BackgroundJobSettings.SectionName}:UseDurableQueue is true but ConnectionStrings:DefaultConnection is not configured.");
+
+        builder.Services.AddHangfire(config => config
+            .SetDataCompatibilityLevel(CompatibilityLevel.Version_180)
+            .UseSimpleAssemblyNameTypeSerializer()
+            .UseRecommendedSerializerSettings()
+            .UsePostgreSqlStorage(postgres => postgres.UseNpgsqlConnection(connectionString)));
+        builder.Services.AddHangfireServer();
+
+        builder.Services.AddSingleton<ScheduledJobRunner>();
+        AddScheduledJob<ReminderSchedulerService>(builder);
+        AddScheduledJob<GoalDeadlineNotificationService>(builder);
+        AddScheduledJob<SlipAlertSchedulerService>(builder);
+        AddScheduledJob<AccountDeletionService>(builder);
+        AddScheduledJob<HabitDueDateAdvancementService>(builder);
+        AddScheduledJob<StreakGoalSyncService>(builder);
+        AddScheduledJob<StreakFreezeAutoActivationService>(builder);
+        AddScheduledJob<SyncCleanupService>(builder);
+        AddScheduledJob<PlayNotificationCleanupService>(builder);
+        AddScheduledJob<CalendarAutoSyncService>(builder);
+
+        builder.Services.AddHostedService<HangfireRecurringJobRegistrar>();
+    }
+
+    private static void AddScheduledJob<TJob>(WebApplicationBuilder builder)
+        where TJob : class, IScheduledJob
+    {
+        builder.Services.AddSingleton<TJob>();
+        builder.Services.AddSingleton<IScheduledJob>(sp => sp.GetRequiredService<TJob>());
+    }
+
+    private static void AddOrbitDistributedCache(WebApplicationBuilder builder)
+    {
+        var redisSettings = builder.Configuration.GetSection(RedisCacheSettings.SectionName).Get<RedisCacheSettings>()
+            ?? new RedisCacheSettings();
+
+        if (!redisSettings.Enabled)
+        {
+            builder.Services.AddDistributedMemoryCache();
+            return;
+        }
+
+        if (string.IsNullOrWhiteSpace(redisSettings.ConnectionString))
+            throw new InvalidOperationException(
+                $"{RedisCacheSettings.SectionName}:Enabled is true but {RedisCacheSettings.SectionName}:ConnectionString is not configured.");
+
+        builder.Services.AddStackExchangeRedisCache(options =>
+        {
+            options.Configuration = redisSettings.ConnectionString;
+            options.InstanceName = redisSettings.InstanceName;
+        });
     }
 
     private static void AddCorsPolicies(WebApplicationBuilder builder)

src/Orbit.Api/Orbit.Api.csproj

@@ -11,7 +11,10 @@
   </ItemGroup>
 
   <ItemGroup>
+    <PackageReference Include="Hangfire.AspNetCore" Version="1.8.23" />
+    <PackageReference Include="Hangfire.PostgreSql" Version="1.21.1" />
     <PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="10.0.2" />
+    <PackageReference Include="Microsoft.Extensions.Caching.StackExchangeRedis" Version="10.0.2" />
     <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="10.0.2">
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
       <PrivateAssets>all</PrivateAssets>

src/Orbit.Api/appsettings.json

@@ -78,7 +78,13 @@
     "GoalDeadlineIntervalMinutes": 30,
     "SlipAlertIntervalMinutes": 5,
     "AccountDeletionIntervalHours": 24,
-    "DueDateAdvancementIntervalMinutes": 30
+    "DueDateAdvancementIntervalMinutes": 30,
+    "UseDurableQueue": false
+  },
+  "Redis": {
+    "Enabled": false,
+    "ConnectionString": "",
+    "InstanceName": "orbit:"
   },
   "Sentry": {
     "Dsn": "",

src/Orbit.Application/Profile/Commands/SetTimezoneCommand.cs

@@ -29,7 +29,7 @@ public async Task<Result> Handle(SetTimezoneCommand request, CancellationToken c
             return result;
 
         await unitOfWork.SaveChangesAsync(cancellationToken);
-        userDateService.InvalidateUserDatePreferences(request.UserId);
+        await userDateService.InvalidateUserDatePreferencesAsync(request.UserId, cancellationToken);
 
         return Result.Success();
     }

src/Orbit.Application/Profile/Commands/SetWeekStartDayCommand.cs

@@ -29,7 +29,7 @@ public async Task<Result> Handle(SetWeekStartDayCommand request, CancellationTok
             return result;
 
         await unitOfWork.SaveChangesAsync(cancellationToken);
-        userDateService.InvalidateUserDatePreferences(request.UserId);
+        await userDateService.InvalidateUserDatePreferencesAsync(request.UserId, cancellationToken);
 
         return Result.Success();
     }

src/Orbit.Domain/Interfaces/IUserDateService.cs

@@ -16,5 +16,5 @@ public interface IUserDateService
     /// from any command that mutates User.TimeZone or User.WeekStartDay, otherwise subsequent
     /// date math can lag by up to 15 minutes.
     /// </summary>
-    void InvalidateUserDatePreferences(Guid userId);
+    Task InvalidateUserDatePreferencesAsync(Guid userId, CancellationToken cancellationToken = default);
 }

src/Orbit.Infrastructure/BackgroundJobs/HangfireRecurringJobRegistrar.cs

@@ -0,0 +1,37 @@
+using Hangfire;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+
+namespace Orbit.Infrastructure.BackgroundJobs;
+
+/// <summary>
+/// Registers every <see cref="IScheduledJob"/> as a Hangfire recurring job on startup when the
+/// durable-queue flag is on. Each job is keyed by its stable name, so re-registering on every boot
+/// reconciles cron changes without creating duplicates, and Hangfire's storage keeps the schedule
+/// across restarts while its distributed lock ensures a single instance runs each occurrence.
+/// </summary>
+public sealed partial class HangfireRecurringJobRegistrar(
+    IRecurringJobManager recurringJobManager,
+    IEnumerable<IScheduledJob> jobs,
+    ILogger<HangfireRecurringJobRegistrar> logger) : IHostedService
+{
+    public Task StartAsync(CancellationToken cancellationToken)
+    {
+        foreach (var job in jobs)
+        {
+            recurringJobManager.AddOrUpdate<ScheduledJobRunner>(
+                job.Name,
+                runner => runner.RunAsync(job.Name, CancellationToken.None),
+                job.CronExpression);
+
+            LogJobRegistered(logger, job.Name, job.CronExpression);
+        }
+
+        return Task.CompletedTask;
+    }
+
+    public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask;
+
+    [LoggerMessage(EventId = 1, Level = LogLevel.Information, Message = "Registered durable recurring job {JobName} with schedule {CronExpression}")]
+    private static partial void LogJobRegistered(ILogger logger, string jobName, string cronExpression);
+}

src/Orbit.Infrastructure/BackgroundJobs/IScheduledJob.cs

@@ -0,0 +1,17 @@
+namespace Orbit.Infrastructure.BackgroundJobs;
+
+/// <summary>
+/// A recurring background scan that can run either as an in-process polling loop or as a durable
+/// Hangfire recurring job, selected by the <c>BackgroundServices:UseDurableQueue</c> flag. Each
+/// scheduler exposes its stable <see cref="Name"/> (the Hangfire recurring-job id and lock key) and
+/// the <see cref="CronExpression"/> that mirrors its default in-process interval, while
+/// <see cref="RunAsync"/> performs one occurrence of the same work the polling loop runs each tick.
+/// </summary>
+public interface IScheduledJob
+{
+    string Name { get; }
+
+    string CronExpression { get; }
+
+    Task RunAsync(CancellationToken cancellationToken);
+}

src/Orbit.Infrastructure/BackgroundJobs/ScheduledJobRunner.cs

@@ -0,0 +1,18 @@
+namespace Orbit.Infrastructure.BackgroundJobs;
+
+/// <summary>
+/// Single Hangfire entry point for every <see cref="IScheduledJob"/>. Hangfire persists only the
+/// job's <see cref="IScheduledJob.Name"/> in storage and resolves a fresh runner per execution, so
+/// adding or renaming a job never changes the serialized recurring-job payload. The runner looks up
+/// the matching job by name and executes one occurrence of its work.
+/// </summary>
+public sealed class ScheduledJobRunner(IEnumerable<IScheduledJob> jobs)
+{
+    public Task RunAsync(string jobName, CancellationToken cancellationToken)
+    {
+        var job = jobs.FirstOrDefault(candidate => candidate.Name == jobName)
+            ?? throw new InvalidOperationException($"No scheduled job registered with name '{jobName}'.");
+
+        return job.RunAsync(cancellationToken);
+    }
+}

src/Orbit.Infrastructure/Configuration/BackgroundJobSettings.cs

@@ -0,0 +1,16 @@
+namespace Orbit.Infrastructure.Configuration;
+
+/// <summary>
+/// Background-processing rollout settings. When <see cref="UseDurableQueue"/> is false (default),
+/// each recurring scheduler runs as its own in-process <c>BackgroundService</c> polling loop exactly
+/// as before. When true, those recurring scans are registered as Hangfire recurring jobs backed by
+/// PostgreSQL instead: occurrences survive restarts, a distributed lock prevents more than one
+/// instance running the same occurrence, and failed runs retry with exponential backoff. The
+/// one-shot startup data-encryption migration always runs as a hosted service regardless of this flag.
+/// </summary>
+public sealed class BackgroundJobSettings
+{
+    public const string SectionName = "BackgroundServices";
+
+    public bool UseDurableQueue { get; init; }
+}

src/Orbit.Infrastructure/Configuration/RedisCacheSettings.cs

@@ -0,0 +1,18 @@
+namespace Orbit.Infrastructure.Configuration;
+
+/// <summary>
+/// Distributed-cache rollout settings. When <see cref="Enabled"/> is false (default), the app
+/// registers an in-process <c>IDistributedCache</c> and behaves exactly as before. When true,
+/// the same <c>IDistributedCache</c> seam is backed by Redis so cached user-date preferences stay
+/// consistent across multiple API instances. <see cref="ConnectionString"/> is required when enabled.
+/// </summary>
+public sealed class RedisCacheSettings
+{
+    public const string SectionName = "Redis";
+
+    public bool Enabled { get; init; }
+
+    public string ConnectionString { get; init; } = "";
+
+    public string InstanceName { get; init; } = "orbit:";
+}

src/Orbit.Infrastructure/Orbit.Infrastructure.csproj

@@ -8,6 +8,7 @@
   <ItemGroup>
     <PackageReference Include="FileSignatures" Version="7.2.1" />
     <PackageReference Include="FirebaseAdmin" Version="3.5.0" />
+    <PackageReference Include="Hangfire.Core" Version="1.8.23" />
     <PackageReference Include="Google.Apis.AndroidPublisher.v3" Version="1.74.0.4165" />
     <PackageReference Include="Google.Apis.Calendar.v3" Version="1.73.0.4073" />
     <PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="10.0.9" />

src/Orbit.Infrastructure/Services/AccountDeletionService.cs

@@ -4,18 +4,30 @@
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
 using Orbit.Domain.Interfaces;
+using Orbit.Infrastructure.BackgroundJobs;
 using Orbit.Infrastructure.Persistence;
 
 namespace Orbit.Infrastructure.Services;
 
 public partial class AccountDeletionService(
     IServiceScopeFactory scopeFactory,
     ILogger<AccountDeletionService> logger,
-    IConfiguration configuration) : BackgroundService
+    IConfiguration configuration) : BackgroundService, IScheduledJob
 {
     private readonly TimeSpan _interval = TimeSpan.FromHours(
         configuration.GetValue("BackgroundServices:AccountDeletionIntervalHours", 24));
 
+    public string Name => "account-deletion";
+
+    public string CronExpression => "0 3 * * *";
+
+    public async Task RunAsync(CancellationToken cancellationToken)
+    {
+        await ProcessScheduledDeletions(cancellationToken);
+        await CleanupStaleSentRecords(cancellationToken);
+        BackgroundServiceHealthCheck.RecordTick("AccountDeletion");
+    }
+
     protected override async Task ExecuteAsync(CancellationToken stoppingToken)
     {
         LogServiceStarted(logger);

src/Orbit.Infrastructure/Services/BackgroundServiceHealthCheck.cs

@@ -17,7 +17,8 @@ public class BackgroundServiceHealthCheck : IHealthCheck
         ["StreakFreezeAutoActivation"] = TimeSpan.FromMinutes(180),
         ["AccountDeletion"] = TimeSpan.FromHours(72),
         ["SyncCleanup"] = TimeSpan.FromHours(48),
-        ["PlayNotificationCleanup"] = TimeSpan.FromHours(48)
+        ["PlayNotificationCleanup"] = TimeSpan.FromHours(48),
+        ["CalendarAutoSync"] = TimeSpan.FromMinutes(45)
     };
 
     public static void RecordTick(string serviceName)

src/Orbit.Infrastructure/Services/CalendarAutoSyncService.cs

@@ -6,6 +6,7 @@
 using Microsoft.Extensions.Logging;
 using Orbit.Application.Calendar.Commands;
 using Orbit.Domain.Enums;
+using Orbit.Infrastructure.BackgroundJobs;
 using Orbit.Infrastructure.Persistence;
 
 namespace Orbit.Infrastructure.Services;
@@ -19,13 +20,23 @@ public partial class CalendarAutoSyncService(
     IServiceScopeFactory scopeFactory,
     ILogger<CalendarAutoSyncService> logger,
     IConfiguration configuration,
-    TimeProvider timeProvider) : BackgroundService
+    TimeProvider timeProvider) : BackgroundService, IScheduledJob
 {
     private readonly TimeSpan _interval = TimeSpan.FromMinutes(
         configuration.GetValue("BackgroundServices:CalendarAutoSyncIntervalMinutes", 15));
     private static readonly TimeSpan DedupeWindow = TimeSpan.FromHours(4);
     private const int BatchSize = 50;
 
+    public string Name => "calendar-auto-sync";
+
+    public string CronExpression => "*/15 * * * *";
+
+    public async Task RunAsync(CancellationToken cancellationToken)
+    {
+        await ProcessTick(cancellationToken);
+        BackgroundServiceHealthCheck.RecordTick("CalendarAutoSync");
+    }
+
     protected override async Task ExecuteAsync(CancellationToken stoppingToken)
     {
         LogServiceStarted(logger);

src/Orbit.Infrastructure/Services/GoalDeadlineNotificationService.cs

@@ -8,20 +8,31 @@
 using Orbit.Domain.Entities;
 using Orbit.Domain.Enums;
 using Orbit.Domain.Interfaces;
+using Orbit.Infrastructure.BackgroundJobs;
 using Orbit.Infrastructure.Persistence;
 
 namespace Orbit.Infrastructure.Services;
 
 public partial class GoalDeadlineNotificationService(
     IServiceScopeFactory scopeFactory,
     ILogger<GoalDeadlineNotificationService> logger,
-    IConfiguration configuration) : BackgroundService
+    IConfiguration configuration) : BackgroundService, IScheduledJob
 {
     private static readonly int[] NotifyDaysBefore = [7, 3, 1];
 
     private readonly TimeSpan _interval = TimeSpan.FromMinutes(
         configuration.GetValue("BackgroundServices:GoalDeadlineIntervalMinutes", 30));
 
+    public string Name => "goal-deadline-notification";
+
+    public string CronExpression => "*/30 * * * *";
+
+    public async Task RunAsync(CancellationToken cancellationToken)
+    {
+        await CheckAndSendDeadlineNotifications(cancellationToken);
+        BackgroundServiceHealthCheck.RecordTick("GoalDeadlineNotification");
+    }
+
     protected override async Task ExecuteAsync(CancellationToken stoppingToken)
     {
         LogServiceStarted(logger);