test(infra): cover external SDK/HTTP failure modes for Resend, FCM, and OpenAI

src/Orbit.Infrastructure/Services/PushNotificationService.cs

@@ -55,7 +55,7 @@
         }
     }
 
     private async Task SendFcm(
         List<Domain.Entities.PushSubscription> subs,
         string title, string body, string? url,
         List<Domain.Entities.PushSubscription> staleSubscriptions,
@@ -101,10 +101,7 @@
                 var tokenPreview = sub.Endpoint[..Math.Min(20, sub.Endpoint.Length)] + "...";
                 if (resp.IsSuccess) continue;
 
-                if (resp.Exception is FirebaseMessagingException fme && (
-                    fme.MessagingErrorCode == MessagingErrorCode.Unregistered ||
-                    fme.MessagingErrorCode == MessagingErrorCode.InvalidArgument ||
-                    fme.MessagingErrorCode == MessagingErrorCode.SenderIdMismatch))
+                if (resp.Exception is FirebaseMessagingException fme && IsStaleFcmError(fme.MessagingErrorCode))
                 {
                     if (logger.IsEnabled(LogLevel.Debug))
                         LogFcmTokenStale(logger, tokenPreview);
@@ -137,10 +134,7 @@
             if (logger.IsEnabled(LogLevel.Debug))
                 LogFcmPushSent(logger, tokenPreview);
         }
-        catch (FirebaseMessagingException ex) when (
-            ex.MessagingErrorCode == MessagingErrorCode.Unregistered ||
-            ex.MessagingErrorCode == MessagingErrorCode.InvalidArgument ||
-            ex.MessagingErrorCode == MessagingErrorCode.SenderIdMismatch)
+        catch (FirebaseMessagingException ex) when (IsStaleFcmError(ex.MessagingErrorCode))
         {
             if (logger.IsEnabled(LogLevel.Debug))
                 LogFcmTokenStale(logger, tokenPreview);
@@ -173,7 +167,7 @@
             await SendWebPushToSubscription(client, sub, message, staleSubscriptions, ct);
     }
 
     private async Task SendWebPushToSubscription(
         PushServiceClient client,
         Domain.Entities.PushSubscription sub,
         PushMessage message,
@@ -193,7 +187,7 @@
             }
         };
 
         for (var attempt = 0; ; attempt++)
         {
             try
             {
@@ -233,6 +227,19 @@
         }
     }
 
+    /// <summary>
+    /// A push token is pruned only when FCM reports it permanently invalid: an unregistered device,
+    /// a malformed token, or a sender-ID mismatch. Transient failures - rate limits
+    /// (<see cref="MessagingErrorCode.QuotaExceeded"/>), upstream auth faults
+    /// (<see cref="MessagingErrorCode.ThirdPartyAuthError"/>), and server errors
+    /// (<see cref="MessagingErrorCode.Internal"/> / <see cref="MessagingErrorCode.Unavailable"/>) -
+    /// keep the subscription so a later delivery can succeed, isolating one bad send from the batch.
+    /// </summary>
+    internal static bool IsStaleFcmError(MessagingErrorCode? code) =>
+        code is MessagingErrorCode.Unregistered
+            or MessagingErrorCode.InvalidArgument
+            or MessagingErrorCode.SenderIdMismatch;
+
     private static bool IsTransient(HttpStatusCode? status) =>
         status is null
             or HttpStatusCode.RequestTimeout

tests/Orbit.Infrastructure.Tests/AI/AiCompletionErrorTests.cs

@@ -0,0 +1,126 @@
+using System.ClientModel;
+using System.ClientModel.Primitives;
+using System.Net;
+using System.Text;
+using FluentAssertions;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Logging.Abstractions;
+using NSubstitute;
+using OpenAI;
+using OpenAI.Chat;
+using Orbit.Domain.Interfaces;
+using Orbit.Infrastructure.AI;
+
+namespace Orbit.Infrastructure.Tests.AI;
+
+/// <summary>
+/// Failure-mode coverage for <see cref="AiCompletionClient"/> against OpenAI HTTP error responses.
+/// The documented contract is to propagate the API failure to the caller (the trust boundary) while
+/// the retry-logging policy makes each attempt observable and no phantom token usage is recorded for
+/// a call that never produced a completion.
+/// </summary>
+public class AiCompletionErrorTests
+{
+    [Fact]
+    public async Task CompleteTextAsync_QuotaExhausted429_RetriesThenThrowsWithoutRecordingUsage()
+    {
+        const string quotaBody =
+            "{\"error\":{\"message\":\"You exceeded your current quota\",\"type\":\"insufficient_quota\",\"code\":\"insufficient_quota\"}}";
+        var handler = new FixedStatusHandler(HttpStatusCode.TooManyRequests, quotaBody);
+        var retryLogger = new CollectingLogger();
+        var usageRecorder = Substitute.For<IAiUsageRecorder>();
+        var client = BuildClient(handler, retryLogger, usageRecorder, maxRetries: 2);
+
+        var act = () => client.CompleteTextAsync("system", "user");
+
+        var exception = (await act.Should().ThrowAsync<ClientResultException>()).Which;
+        exception.Status.Should().Be(429);
+        handler.CallCount.Should().Be(3);
+        retryLogger.Entries.Should().HaveCount(3);
+        retryLogger.Entries[^1].Should().Contain("retriable: False");
+        await usageRecorder.DidNotReceive().RecordAsync(
+            Arg.Any<string>(), Arg.Any<string>(), Arg.Any<long>(), Arg.Any<long>(),
+            Arg.Any<long>(), Arg.Any<long>(), Arg.Any<CancellationToken>());
+    }
+
+    [Fact]
+    public async Task CompleteJsonAsync_ContextLengthExceeded400_ThrowsImmediatelyWithoutRetryOrUsage()
+    {
+        const string contextBody =
+            "{\"error\":{\"message\":\"maximum context length exceeded\",\"type\":\"invalid_request_error\",\"code\":\"context_length_exceeded\"}}";
+        var handler = new FixedStatusHandler(HttpStatusCode.BadRequest, contextBody);
+        var retryLogger = new CollectingLogger();
+        var usageRecorder = Substitute.For<IAiUsageRecorder>();
+        var client = BuildClient(handler, retryLogger, usageRecorder, maxRetries: 2);
+
+        var act = () => client.CompleteJsonAsync<Probe>("system", "user");
+
+        var exception = (await act.Should().ThrowAsync<ClientResultException>()).Which;
+        exception.Status.Should().Be(400);
+        handler.CallCount.Should().Be(1);
+        retryLogger.Entries.Should().ContainSingle().Which.Should().Contain("retriable: False");
+        await usageRecorder.DidNotReceive().RecordAsync(
+            Arg.Any<string>(), Arg.Any<string>(), Arg.Any<long>(), Arg.Any<long>(),
+            Arg.Any<long>(), Arg.Any<long>(), Arg.Any<CancellationToken>());
+    }
+
+    private static AiCompletionClient BuildClient(
+        HttpMessageHandler handler, ILogger retryLogger, IAiUsageRecorder usageRecorder, int maxRetries)
+    {
+        var chatClient = new ChatClient(
+            model: "gpt-test",
+            credential: new ApiKeyCredential("test-key"),
+            options: new OpenAIClientOptions
+            {
+                Endpoint = new Uri("https://orbit.test/v1"),
+                Transport = new HttpClientPipelineTransport(new HttpClient(handler)),
+                RetryPolicy = new ZeroDelayRetryPolicy(maxRetries, retryLogger),
+            });
+        return new AiCompletionClient(chatClient, NullLogger<AiCompletionClient>.Instance, usageRecorder);
+    }
+
+    private sealed record Probe(string Value);
+
+    private sealed class ZeroDelayRetryPolicy(int maxRetries, ILogger logger)
+        : AiRetryLoggingPolicy(maxRetries, logger)
+    {
+        protected override TimeSpan GetNextDelay(PipelineMessage message, int tryCount) => TimeSpan.Zero;
+    }
+
+    private sealed class FixedStatusHandler(HttpStatusCode status, string body) : HttpMessageHandler
+    {
+        public int CallCount { get; private set; }
+
+        protected override HttpResponseMessage Send(HttpRequestMessage request, CancellationToken cancellationToken)
+            => BuildResponse(request);
+
+        protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
+            => Task.FromResult(BuildResponse(request));
+
+        private HttpResponseMessage BuildResponse(HttpRequestMessage request)
+        {
+            CallCount++;
+            return new HttpResponseMessage(status)
+            {
+                RequestMessage = request,
+                Content = new StringContent(body, Encoding.UTF8, "application/json"),
+            };
+        }
+    }
+
+    private sealed class CollectingLogger : ILogger
+    {
+        public List<string> Entries { get; } = [];
+
+        public IDisposable? BeginScope<TState>(TState state) where TState : notnull => null;
+
+        public bool IsEnabled(LogLevel logLevel) => true;
+
+        public void Log<TState>(
+            LogLevel logLevel,
+            EventId eventId,
+            TState state,
+            Exception? exception,
+            Func<TState, Exception?, string> formatter) => Entries.Add(formatter(state, exception));
+    }
+}

tests/Orbit.Infrastructure.Tests/Services/PushNotificationServiceTests.cs

@@ -1,6 +1,7 @@
 using System.Net;
 using System.Security.Cryptography;
 using System.Text;
+using FirebaseAdmin.Messaging;
 using FluentAssertions;
 using Microsoft.Data.Sqlite;
 using Microsoft.EntityFrameworkCore;
@@ -245,6 +246,22 @@ public async Task SendToUserAsync_HostileOverlongContent_DeliversPayloadUnderPla
         (await _dbContext.PushSubscriptions.CountAsync()).Should().Be(1);
     }
 
+    [Theory]
+    [InlineData(MessagingErrorCode.QuotaExceeded)]
+    [InlineData(MessagingErrorCode.ThirdPartyAuthError)]
+    [InlineData(MessagingErrorCode.Internal)]
+    [InlineData(MessagingErrorCode.Unavailable)]
+    [InlineData(null)]
+    public void IsStaleFcmError_RateLimitAuthAndTransientErrors_KeepsSubscription(MessagingErrorCode? code) =>
+        PushNotificationService.IsStaleFcmError(code).Should().BeFalse();
+
+    [Theory]
+    [InlineData(MessagingErrorCode.Unregistered)]
+    [InlineData(MessagingErrorCode.InvalidArgument)]
+    [InlineData(MessagingErrorCode.SenderIdMismatch)]
+    public void IsStaleFcmError_PermanentlyInvalidTokens_PrunesSubscription(MessagingErrorCode code) =>
+        PushNotificationService.IsStaleFcmError(code).Should().BeTrue();
+
     [Fact]
     public void SanitizeForDelivery_StripsControlCharacters()
     {

tests/Orbit.Infrastructure.Tests/Services/ResendEmailServiceMarketingRetryTests.cs

@@ -75,6 +75,16 @@ public async Task DoesNotRetry_OnNonRetriable4xx()
         handler.CallCount.Should().Be(1);
     }
 
+    [Fact]
+    public async Task DoesNotRetry_On403Forbidden()
+    {
+        var (sut, handler) = Build(HttpStatusCode.Forbidden);
+
+        await Send(sut);
+
+        handler.CallCount.Should().Be(1);
+    }
+
     private sealed class SequenceHandler(params HttpStatusCode[] responses) : HttpMessageHandler
     {
         private readonly Queue<HttpStatusCode> _responses = new(responses);

tests/Orbit.Infrastructure.Tests/Services/ResendEmailServiceTests.cs

@@ -83,6 +83,47 @@ public async Task SendVerificationCodeAsync_ApiFailure_DoesNotThrow()
         await act.Should().NotThrowAsync();
     }
 
+    [Fact]
+    public async Task SendVerificationCodeAsync_AuthError401_HandledWithoutRetry()
+    {
+        _handler.ResponseToReturn = new HttpResponseMessage(HttpStatusCode.Unauthorized)
+        {
+            Content = new StringContent("{\"message\":\"Invalid API key\"}")
+        };
+
+        var act = () => _sut.SendVerificationCodeAsync("real.user@example.com", "123456");
+
+        await act.Should().NotThrowAsync();
+        _handler.LastRequest.Should().NotBeNull();
+        _handler.CallCount.Should().Be(1);
+    }
+
+    [Fact]
+    public async Task SendVerificationCodeAsync_InvalidEmailAddress422_HandledWithoutRetry()
+    {
+        _handler.ResponseToReturn = new HttpResponseMessage(HttpStatusCode.UnprocessableEntity)
+        {
+            Content = new StringContent("{\"name\":\"validation_error\",\"message\":\"Invalid `to` field\"}")
+        };
+
+        var act = () => _sut.SendVerificationCodeAsync("not-an-email", "123456");
+
+        await act.Should().NotThrowAsync();
+        _handler.LastRequest.Should().NotBeNull();
+        _handler.CallCount.Should().Be(1);
+    }
+
+    [Fact]
+    public async Task SendVerificationCodeAsync_TransientServerError_RetriesUpToPolicyLimit()
+    {
+        _handler.ResponseFactory = () => new HttpResponseMessage(HttpStatusCode.ServiceUnavailable);
+
+        var act = () => _sut.SendVerificationCodeAsync("real.user@example.com", "123456");
+
+        await act.Should().NotThrowAsync();
+        _handler.CallCount.Should().Be(1 + Orbit.Infrastructure.Common.HttpRetryPolicy.MaxRetries);
+    }
+
     [Fact]
     public async Task SendVerificationCodeAsync_HttpException_DoesNotThrow()
     {
@@ -348,21 +389,25 @@ await _sut.SendWaitlistConfirmationAsync(
     private class FakeHttpMessageHandler : HttpMessageHandler
     {
         public HttpResponseMessage ResponseToReturn { get; set; } = new(HttpStatusCode.OK);
+        public Func<HttpResponseMessage>? ResponseFactory { get; set; }
         public Exception? ExceptionToThrow { get; set; }
         public HttpRequestMessage? LastRequest { get; private set; }
         public string LastRequestBody { get; private set; } = "";
+        public int CallCount { get; private set; }
 
         protected override async Task<HttpResponseMessage> SendAsync(
             HttpRequestMessage request, CancellationToken cancellationToken)
         {
+            CallCount++;
+
             if (ExceptionToThrow is not null)
                 throw ExceptionToThrow;
 
             LastRequest = request;
             if (request.Content is not null)
                 LastRequestBody = await request.Content.ReadAsStringAsync(cancellationToken);
 
-            return ResponseToReturn;
+            return ResponseFactory?.Invoke() ?? ResponseToReturn;
         }
     }
 }