diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json index 9b5216716..b8f9a3018 100644 --- a/.config/dotnet-tools.json +++ b/.config/dotnet-tools.json @@ -3,10 +3,10 @@ "isRoot": true, "tools": { "dotnet-stryker": { - "version": "4.14.1", + "version": "4.14.2", "commands": [ "dotnet-stryker" ] } } -} +} \ No newline at end of file diff --git a/Directory.Packages.props b/Directory.Packages.props index 25c224420..7a0b9360e 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -5,10 +5,10 @@ - + - + @@ -35,7 +35,7 @@ - + @@ -76,7 +76,7 @@ - + diff --git a/docs/sections/Mailer.md b/docs/sections/Mailer.md index 4bdc9b6c4..95888077a 100644 --- a/docs/sections/Mailer.md +++ b/docs/sections/Mailer.md @@ -5,8 +5,9 @@ Orchestrates Humans ↔ MailerLite synchronisation. Inbound import + outbound au ## Concepts - **MailerLite subscriber** — a row in ML's `subscribers` collection. Has `status ∈ {active, unsubscribed, unconfirmed, bounced, junk}` and `subscribed_at` / `unsubscribed_at` / `opted_in_at` timestamps. Tracks the `groups` the subscriber belongs to. -- **Import plan** — the classified result of pulling every ML subscriber and matching against Humans's user/email/preference state. Built fresh on every preview/commit; never persisted between runs. -- **Apply** — executes an import plan: creates contacts, attaches verified users, deletes unverified UserEmail rows that block contact creation, updates Marketing preferences per the conflict rule, writes one summary audit. +- **Import plan** — the classified result of pulling the MailerLite `Website` group's subscribers and matching against Humans's user/email/preference state, plus a Humans-side pass that flags Marketing opt-ins the prior whole-account import wrongly set (see **Reset**). Built fresh on every preview/commit; never persisted between runs. +- **Apply** — executes an import plan: creates contacts, attaches verified users, deletes unverified UserEmail rows that block contact creation, updates Marketing preferences per the conflict rule, resets (deletes → null) Marketing flags caught by the reset pass, writes one summary audit. +- **Reset (marketing flag)** — a Humans-side cleanup decision: a Marketing opt-in written by the erroneous whole-account import (`UpdateSource = "MailerLiteSync"`, opted-in, no prior consent) on someone **not** in the `Website` group is deleted, reverting the category to "no preference" (null) — never to opt-out. GDPR remediation; cutoff for "prior consent" is hardcoded in `MailerImportService.BadImportCutoff`. - **Audience** — a code-defined `IMailerAudience` implementation whose `MailerLiteGroupName` starts with `"Humans - "`. Membership is computed from Humans state and synced into the ML group by `MailerAudienceSyncService` (daily Hangfire job + on-demand admin button). ## Data Model @@ -36,7 +37,8 @@ All routes are `AdminOnly`. - Every outbound write targets an ML group whose `Name` starts with `"Humans - "`. `MailerLiteClient` runtime-rejects writes against non-`"Humans - "` groups with `InvalidOperationException`. Pinned by `MailerLiteClientWriteGuardTests`. - All `IMailerAudience` implementations target group names starting with `"Humans - "`. Pinned by `MailerArchitectureTests.AllAudiences_UseHumansPrefix`. Audience keys and group names are unique across registrations (pinned by `AllAudiences_HaveUniqueGroupNamesAndKeys`). - `MailerImportService` and `MailerAudienceSyncService` live in `Humans.Application` and never reference `Microsoft.EntityFrameworkCore`. Pinned by architecture tests. -- Every write to `CommunicationPreference[Marketing]` goes through `CommunicationPreferenceService.UpdatePreferenceAsync` and produces a `CommunicationPreferenceChanged` audit entry on real state changes (not idempotent confirms). +- The import (`MailerImportService`) ingests **only** the MailerLite group named `Website` (resolved by name; throws if the group is absent) — never the whole account. The reset pass excludes anyone in that group of any status, since the import already owns their pref (active → opt-in, unsubscribed/bounced → opt-out). Pinned by `MailerImportServiceWebsiteScopeTests`. +- Every write to `CommunicationPreference[Marketing]` goes through `CommunicationPreferenceService` — `UpdatePreferenceAsync` for opt state, `ResetPreferenceAsync` to delete the row (→ null) — and produces a `CommunicationPreferenceChanged` audit entry on real state changes (not idempotent confirms). - `ApplyAsync` is idempotent: a second run against unchanged ML+Humans state writes zero per-row entries and exactly one `MailerLiteReconciliationCompleted` summary entry. - `SyncAsync` is idempotent: a second run against unchanged audience+ML state writes zero ML mutations and exactly one `MailerLiteAudienceSyncCompleted` summary entry whose counts are all zero. - Bounced/junk subscribers always set `OptedOut = true` regardless of any Humans-side timestamp. Delivery facts override preferences. @@ -62,9 +64,9 @@ All routes are `AdminOnly`. - **Profiles**: reads `IUserEmailService.FindVerifiedEmailWithUserAsync`, `FindAnyUserIdByEmailAsync`, `DeleteEmailAsync`, `GetPrimaryEmailsByUserIdsAsync`; reads/writes `ICommunicationPreferenceService.GetAsync` / `UpdatePreferenceAsync` / `GetCountByCategoryAndStateAsync`. - **Users**: writes via `IAccountProvisioningService.FindOrCreateUserByEmailAsync`; reads `IUserService.GetByIdAsync` (tombstone follow), `IUserService.GetCountByContactSourceAsync`. -- **Tickets**: `ITicketQueryService.GetUserIdsWithTicketsAsync` — audience-side ticket-holder enumeration for `TicketNoShiftsAudience` and `HasTicketAudience`. Scoped to the active vendor event by the cached decorator (see `TicketSyncState.VendorEventId`). +- **Tickets**: `ITicketQueryService.GetUserIdsWithTicketsAsync` — audience-side ticket-holder enumeration for `TicketNoShiftsAudience`, `HasTicketAudience`, and `MarketingNoTicketAudience`. Scoped to the active vendor event by the cached decorator (see `TicketSyncState.VendorEventId`). - **Shifts**: `IShiftView.GetUsersAsync` + `IUserService.GetAllUserInfosAsync` — cached per-user shift signups, used by `TicketNoShiftsAudience` and `HasShiftAudience` (encode Pending/Confirmed-on-active-event via `ShiftUserView.HasShift`). -- **Users**: `IUserService.GetAllUserInfosAsync` — audience-side enumeration of explicit Marketing opt-ins for `MarketingAudience` (`UserInfo.MarketingOptedOut == false`). +- **Users**: `IUserService.GetAllUserInfosAsync` — audience-side enumeration of explicit Marketing opt-ins for `MarketingAudience` and `MarketingNoTicketAudience` (`UserInfo.MarketingOptedOut == false`). - **AuditLog**: writes via `IAuditLogService.LogAsync` (job overload). ## Architecture diff --git a/src/Humans.Application/Interfaces/Mailer/Dtos/ImportPlan.cs b/src/Humans.Application/Interfaces/Mailer/Dtos/ImportPlan.cs index 78ff9b7f4..7b081e188 100644 --- a/src/Humans.Application/Interfaces/Mailer/Dtos/ImportPlan.cs +++ b/src/Humans.Application/Interfaces/Mailer/Dtos/ImportPlan.cs @@ -11,6 +11,7 @@ public sealed record ImportPlan( VerifiedFlipToOptIn: Decisions.Count(d => d.Outcome == SubscriberOutcome.VerifiedFlipToOptIn), VerifiedFlipToOptOut: Decisions.Count(d => d.Outcome == SubscriberOutcome.VerifiedFlipToOptOut), VerifiedKeepHumansPref: Decisions.Count(d => d.Outcome == SubscriberOutcome.VerifiedKeepHumansPref), + ResetMarketingFlag: Decisions.Count(d => d.Outcome == SubscriberOutcome.ResetMarketingFlag), AmbiguousMultipleVerified: Decisions.Count(d => d.Outcome == SubscriberOutcome.AmbiguousMultipleVerified), UnconfirmedSkipped: Decisions.Count(d => d.Outcome == SubscriberOutcome.UnconfirmedSkipped)); } @@ -22,5 +23,6 @@ public sealed record ImportPlanCounts( int VerifiedFlipToOptIn, int VerifiedFlipToOptOut, int VerifiedKeepHumansPref, + int ResetMarketingFlag, int AmbiguousMultipleVerified, int UnconfirmedSkipped); diff --git a/src/Humans.Application/Interfaces/Mailer/Dtos/ImportResult.cs b/src/Humans.Application/Interfaces/Mailer/Dtos/ImportResult.cs index 52240684f..db22aa5fa 100644 --- a/src/Humans.Application/Interfaces/Mailer/Dtos/ImportResult.cs +++ b/src/Humans.Application/Interfaces/Mailer/Dtos/ImportResult.cs @@ -8,6 +8,7 @@ public sealed record ImportResult( int PrefsFlippedToOptIn, int PrefsFlippedToOptOut, int PrefsKeptByConflict, + int MarketingFlagsReset, int UnverifiedEmailsReplaced, int AmbiguousSkipped, int UnconfirmedSkipped, @@ -21,6 +22,7 @@ public string FormatSummary() => $"{HumansCreated} humans created, " + $"{PrefsFlippedToOptIn} flipped to opt-in, {PrefsFlippedToOptOut} flipped to opt-out, " + $"{PrefsKeptByConflict} kept by conflict-rule, " + + $"{MarketingFlagsReset} marketing flags reset, " + $"{UnverifiedEmailsReplaced} unverified emails replaced, " + $"{AmbiguousSkipped} ambiguous skipped, " + $"{UnconfirmedSkipped} unconfirmed skipped, " + diff --git a/src/Humans.Application/Interfaces/Mailer/Dtos/SubscriberDecision.cs b/src/Humans.Application/Interfaces/Mailer/Dtos/SubscriberDecision.cs index 5047afd7e..72d61f99a 100644 --- a/src/Humans.Application/Interfaces/Mailer/Dtos/SubscriberDecision.cs +++ b/src/Humans.Application/Interfaces/Mailer/Dtos/SubscriberDecision.cs @@ -27,6 +27,16 @@ public enum SubscriberOutcome /// Verified human match, user touched Humans' pref more recently than ML — keep Humans' state. VerifiedKeepHumansPref, + + /// + /// Existing human, opted-in to Marketing by the erroneous whole-account + /// MailerLite import (UpdateSource = "MailerLiteSync"), who is not an + /// active member of the Website group and has no genuine prior consent — + /// delete the Marketing pref row to revert to "no preference recorded" + /// (null), not opt-out. GDPR remediation; discovered from the Humans side, + /// not from an ML subscriber. + /// + ResetMarketingFlag, } public sealed record SubscriberDecision( diff --git a/src/Humans.Application/Interfaces/Profiles/ICommunicationPreferenceService.cs b/src/Humans.Application/Interfaces/Profiles/ICommunicationPreferenceService.cs index 148476136..ccb31c390 100644 --- a/src/Humans.Application/Interfaces/Profiles/ICommunicationPreferenceService.cs +++ b/src/Humans.Application/Interfaces/Profiles/ICommunicationPreferenceService.cs @@ -81,6 +81,17 @@ Task UpdatePreferenceAsync( Guid userId, MessageCategory category, bool optedOut, bool inboxEnabled, string source, CancellationToken cancellationToken = default); + /// + /// Deletes the preference row for a user+category, reverting the category to + /// the "no preference recorded" (null) state — distinct from an explicit + /// opt-out. No-op if no row exists. Logs an audit entry when a row is + /// removed. Used by the Mailer import GDPR remediation to undo Marketing + /// opt-ins the erroneous whole-account import set. + /// + Task ResetPreferenceAsync( + Guid userId, MessageCategory category, string source, + CancellationToken cancellationToken = default); + /// /// Generates a time-limited unsubscribe token encoding userId + category. /// Token expires after ~90 days. diff --git a/src/Humans.Application/Interfaces/Repositories/ICommunicationPreferenceRepository.cs b/src/Humans.Application/Interfaces/Repositories/ICommunicationPreferenceRepository.cs index 3c0952656..0a8f2906a 100644 --- a/src/Humans.Application/Interfaces/Repositories/ICommunicationPreferenceRepository.cs +++ b/src/Humans.Application/Interfaces/Repositories/ICommunicationPreferenceRepository.cs @@ -81,6 +81,15 @@ Task> AddDefaultsOrReloadAsync( /// Task UpdateAsync(CommunicationPreference preference, CancellationToken ct = default); + /// + /// Deletes the single communication_preferences row for a user+category, + /// returning the category to its "no row recorded" (null) state — distinct + /// from an explicit opt-out. Returns true if a row was deleted, false if none + /// existed. Used by the Mailer import GDPR remediation. + /// + Task DeleteByUserAndCategoryAsync( + Guid userId, MessageCategory category, CancellationToken ct = default); + /// /// Bulk-moves communication_preferences rows from /// to for the diff --git a/src/Humans.Application/Services/Calendar/CalendarOccurrenceExpander.cs b/src/Humans.Application/Services/Calendar/CalendarOccurrenceExpander.cs index c0a894af3..ebacfe99d 100644 --- a/src/Humans.Application/Services/Calendar/CalendarOccurrenceExpander.cs +++ b/src/Humans.Application/Services/Calendar/CalendarOccurrenceExpander.cs @@ -68,7 +68,7 @@ public static IReadOnlyList Expand( DtStart = new CalDateTime(dtStartLocal, e.RecurrenceTimezone, hasTime: true), Duration = Ical.Net.DataTypes.Duration.FromTimeSpanExact(TimeSpan.FromTicks(dur.BclCompatibleTicks)), }; - icalEv.RecurrenceRules.Add(new RecurrencePattern(e.RecurrenceRule!)); + icalEv.RecurrenceRule = new RecurrencePattern(e.RecurrenceRule!); var fromLocal = from.InZone(zone).LocalDateTime.ToDateTimeUnspecified(); var toLocal = to.InZone(zone).LocalDateTime.ToDateTimeUnspecified(); diff --git a/src/Humans.Application/Services/Calendar/CalendarService.cs b/src/Humans.Application/Services/Calendar/CalendarService.cs index 985c56a37..5977f7934 100644 --- a/src/Humans.Application/Services/Calendar/CalendarService.cs +++ b/src/Humans.Application/Services/Calendar/CalendarService.cs @@ -220,7 +220,7 @@ private static string CalendarValidationMemberName(ValidationException ex) => DtStart = new CalDateTime(dtStartLocal, tz, hasTime: true), Duration = Ical.Net.DataTypes.Duration.FromTimeSpanExact(TimeSpan.FromTicks(duration.BclCompatibleTicks)), }; - icalEv.RecurrenceRules.Add(new RecurrencePattern(rrule)); + icalEv.RecurrenceRule = new RecurrencePattern(rrule); var startCalDt = new CalDateTime(dtStartLocal, tz, hasTime: true); var last = icalEv.GetOccurrences(startCalDt, new EvaluationOptions()) diff --git a/src/Humans.Application/Services/Mailer/Audiences/MarketingNoTicketAudience.cs b/src/Humans.Application/Services/Mailer/Audiences/MarketingNoTicketAudience.cs new file mode 100644 index 000000000..0f316724b --- /dev/null +++ b/src/Humans.Application/Services/Mailer/Audiences/MarketingNoTicketAudience.cs @@ -0,0 +1,30 @@ +using Humans.Application.Interfaces.Mailer; +using Humans.Application.Interfaces.Tickets; +using Humans.Application.Interfaces.Users; + +namespace Humans.Application.Services.Mailer.Audiences; + +/// +/// "Humans - Marketing no Ticket" — humans who have explicitly opted in to the +/// Marketing communication category ( == false) +/// AND do not currently hold a ticket in the active vendor event. +/// Users with no Marketing preference row (default-off) or who hold a ticket are excluded. +/// +public sealed class MarketingNoTicketAudience( + IUserService users, + ITicketQueryService tickets) : IMailerAudience +{ + public string Key => "marketing-no-ticket"; + public string DisplayName => "Marketing opt-ins without a ticket"; + public string MailerLiteGroupName => "Humans - Marketing no Ticket"; + + public async Task> ComputeMemberUserIdsAsync(CancellationToken ct) + { + var ticketHolders = await tickets.GetUserIdsWithTicketsAsync(); + var allUsers = await users.GetAllUserInfosAsync(ct); + return allUsers + .Where(u => u.MarketingOptedOut == false && !ticketHolders.Contains(u.Id)) + .Select(u => u.Id) + .ToHashSet(); + } +} diff --git a/src/Humans.Application/Services/Mailer/MailerImportService.cs b/src/Humans.Application/Services/Mailer/MailerImportService.cs index 4eefcbae8..858bd7244 100644 --- a/src/Humans.Application/Services/Mailer/MailerImportService.cs +++ b/src/Humans.Application/Services/Mailer/MailerImportService.cs @@ -19,11 +19,33 @@ public sealed class MailerImportService( IClock clock, ILogger logger) : IMailerImportService { + // The erroneous import pulled the whole MailerLite account; it must only ever + // ingest the "Website" group. Resolved by name at runtime (group ids aren't + // stable across environments). Reads of this group bypass the client's + // "Humans - " write-guard — it's a source, never written to. + private const string WebsiteGroupName = "Website"; + + // UpdateSource the import stamps on Marketing prefs it writes. + private const string SyncSource = "MailerLiteSync"; + + // Source label recorded on the audit entry when a flag is reset to null. + private const string ResetSource = "MailerLiteSyncReset"; + + // Marketing opt-ins written by the erroneous whole-account import carry no + // genuine prior consent. A Marketing pref whose SubscribedAt predates this + // instant was opted-in before the bad import and is preserved (≤5 known + // cases). Hardcoded — one-time GDPR remediation cutoff. + private static readonly Instant BadImportCutoff = Instant.FromUtc(2026, 5, 19, 0, 1, 1); + public async Task BuildPlanAsync(CancellationToken ct = default) { + var websiteGroupId = await ResolveWebsiteGroupIdAsync(ct); + var decisions = new List(); var subs = new List(); - await foreach (var s in ml.ListSubscribersAsync(ct)) subs.Add(s); + await foreach (var s in ml.ListSubscribersAsync(ct)) + if (s.GroupIds.Contains(websiteGroupId, StringComparer.Ordinal)) + subs.Add(s); foreach (var s in subs) { @@ -67,6 +89,17 @@ public async Task BuildPlanAsync(CancellationToken ct = default) SubscriberOutcome.CreateNewHuman, null, null, null)); } + // GDPR remediation: revert Marketing opt-ins the erroneous whole-account + // import set on people who aren't in the Website group at all. Discovered + // from the Humans side — these users have no Website subscriber row, so they + // never appear in the subscriber loop above. + var websiteEmails = WebsiteSubscriberEmails(subs); + foreach (var u in await users.GetAllUserInfosAsync(ct)) + if (IsResetCandidate(u, websiteEmails)) + decisions.Add(new SubscriberDecision( + u.Email ?? "(no verified email)", "n/a", + SubscriberOutcome.ResetMarketingFlag, u.Id, null, null)); + return new ImportPlan(decisions, subs.Count); } @@ -123,16 +156,22 @@ public async Task ApplyAsync( { var start = clock.GetCurrentInstant(); int created = 0, flippedIn = 0, flippedOut = 0, preserved = 0, - replacedUnverified = 0, vanishedBetweenPlanAndApply = 0, errors = 0; + replacedUnverified = 0, vanishedBetweenPlanAndApply = 0, marketingReset = 0, errors = 0; var pulledIds = new HashSet(StringComparer.Ordinal); - // Re-pull ML so plan/apply are stateless. + var websiteGroupId = await ResolveWebsiteGroupIdAsync(ct); + + // Re-pull ML so plan/apply are stateless. Website group only. var subsByEmail = new Dictionary(StringComparer.OrdinalIgnoreCase); + var websiteSubs = new List(); await foreach (var s in ml.ListSubscribersAsync(ct)) { + if (!s.GroupIds.Contains(websiteGroupId, StringComparer.Ordinal)) continue; subsByEmail[s.Email] = s; + websiteSubs.Add(s); pulledIds.Add(s.Id); } + var websiteEmails = WebsiteSubscriberEmails(websiteSubs); var (toProcess, throttled) = ApplyThrottle(plan.Decisions, maxPerOutcome); @@ -140,6 +179,14 @@ public async Task ApplyAsync( { try { + // Reset candidates aren't ML subscribers — handle before the subsByEmail guard. + if (d.Outcome == SubscriberOutcome.ResetMarketingFlag) + { + if (await TryResetMarketingFlagAsync(d.TargetUserId!.Value, websiteEmails, ct)) + marketingReset++; + continue; + } + if (!subsByEmail.TryGetValue(d.Email, out var subscriber)) { vanishedBetweenPlanAndApply++; @@ -205,6 +252,7 @@ public async Task ApplyAsync( PrefsFlippedToOptIn: flippedIn, PrefsFlippedToOptOut: flippedOut, PrefsKeptByConflict: preserved, + MarketingFlagsReset: marketingReset, UnverifiedEmailsReplaced: replacedUnverified, AmbiguousSkipped: plan.Counts.AmbiguousMultipleVerified, UnconfirmedSkipped: plan.Counts.UnconfirmedSkipped, @@ -278,7 +326,56 @@ private async Task ApplyMarketingDeltaAsync( } await prefs.UpdatePreferenceAsync(userId, MessageCategory.Marketing, - optedOut: mlOptedOut, source: "MailerLiteSync", ct); + optedOut: mlOptedOut, source: SyncSource, ct); return mlOptedOut ? DeltaResult.FlippedToOptOut : DeltaResult.FlippedToOptIn; } + + private async Task ResolveWebsiteGroupIdAsync(CancellationToken ct) + { + var groups = await ml.ListGroupsAsync(ct); + var website = groups.FirstOrDefault(g => + string.Equals(g.Name, WebsiteGroupName, StringComparison.OrdinalIgnoreCase)); + return website?.Id + ?? throw new InvalidOperationException( + $"MailerLite group '{WebsiteGroupName}' not found — refusing to import or reset " + + "(would otherwise sweep the whole account / reset every opted-in human)."); + } + + private static HashSet WebsiteSubscriberEmails(IEnumerable websiteSubs) => + websiteSubs + .Select(s => s.Email) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + + /// + /// A user is a reset candidate when their Marketing pref was opted-in by the + /// erroneous import (), they have no genuine prior + /// consent (a SubscribedAt predating ), + /// and they are not a member of the Website group at all (by any of their + /// emails). Website members — active or not — are owned by the import loop + /// (active → opt-in, unsubscribed/bounced → opt-out), so opt-ins survive only + /// for active members and an explicit unsubscribe stays opt-out rather than + /// being nulled here. Non-members are pure collateral: the row is deleted to + /// revert to "no preference" (null). + /// + private static bool IsResetCandidate(UserInfo user, HashSet websiteEmails) + { + var marketing = user.CommunicationPreferences + .FirstOrDefault(c => c.Category == MessageCategory.Marketing); + if (marketing is null || marketing.OptedOut) return false; + if (!string.Equals(marketing.UpdateSource, SyncSource, StringComparison.Ordinal)) return false; + if (marketing.SubscribedAt is { } subscribedAt && subscribedAt < BadImportCutoff) return false; + return !user.UserEmails.Any(e => websiteEmails.Contains(e.Email)); + } + + // Re-checks the predicate against current cache state so a pref the user set + // between preview and commit isn't clobbered, then deletes the row (→ null). + private async Task TryResetMarketingFlagAsync( + Guid userId, HashSet websiteEmails, CancellationToken ct) + { + var info = await users.GetUserInfoAsync(userId, ct); + if (info is null || !IsResetCandidate(info, websiteEmails)) + return false; + await prefs.ResetPreferenceAsync(userId, MessageCategory.Marketing, ResetSource, ct); + return true; + } } diff --git a/src/Humans.Application/Services/Profiles/CommunicationPreferenceService.cs b/src/Humans.Application/Services/Profiles/CommunicationPreferenceService.cs index 7b28a22cd..282fc2c4b 100644 --- a/src/Humans.Application/Services/Profiles/CommunicationPreferenceService.cs +++ b/src/Humans.Application/Services/Profiles/CommunicationPreferenceService.cs @@ -226,6 +226,31 @@ await auditLog.LogAsync( userId, category, optedOut, inboxEnabled, source); } + public async Task ResetPreferenceAsync( + Guid userId, MessageCategory category, string source, + CancellationToken cancellationToken = default) + { + if (category.IsAlwaysOn()) + { + logger.LogWarning("Attempted to reset always-on preference {Category} for user {UserId} — ignored", category, userId); + return; + } + + var deleted = await repository.DeleteByUserAndCategoryAsync(userId, category, cancellationToken); + if (!deleted) + return; + + await auditLog.LogAsync( + AuditAction.CommunicationPreferenceChanged, + "User", userId, + $"{category} reset to no preference (null) via {source}", + "CommunicationPreferenceService"); + + logger.LogInformation( + "User {UserId} communication preference {Category} reset to null via {Source}", + userId, category, source); + } + public string GenerateUnsubscribeToken(Guid userId, MessageCategory category) => tokenProvider.GenerateToken(userId, category); diff --git a/src/Humans.Infrastructure/Repositories/Profiles/CommunicationPreferenceRepository.cs b/src/Humans.Infrastructure/Repositories/Profiles/CommunicationPreferenceRepository.cs index 4794fb75d..249041a28 100644 --- a/src/Humans.Infrastructure/Repositories/Profiles/CommunicationPreferenceRepository.cs +++ b/src/Humans.Infrastructure/Repositories/Profiles/CommunicationPreferenceRepository.cs @@ -143,6 +143,21 @@ public async Task UpdateAsync(CommunicationPreference preference, CancellationTo await ctx.SaveChangesAsync(ct); } + public async Task DeleteByUserAndCategoryAsync( + Guid userId, MessageCategory category, CancellationToken ct = default) + { + await using var ctx = await factory.CreateDbContextAsync(ct); + var pref = await ctx.CommunicationPreferences + .FirstOrDefaultAsync(cp => cp.UserId == userId && cp.Category == category, ct); + if (pref is null) return false; + + // Tracked Remove → EntityState.Deleted, so UserInfoSaveChangesInterceptor + // refreshes the user's cached preferences after the row is gone. + ctx.CommunicationPreferences.Remove(pref); + await ctx.SaveChangesAsync(ct); + return true; + } + public async Task ReassignToUserAsync( Guid sourceUserId, Guid targetUserId, Instant updatedAt, CancellationToken ct = default) diff --git a/src/Humans.Web/Controllers/AdminDuplicateAccountsController.cs b/src/Humans.Web/Controllers/AdminDuplicateAccountsController.cs index b60b9a5da..b83bfb187 100644 --- a/src/Humans.Web/Controllers/AdminDuplicateAccountsController.cs +++ b/src/Humans.Web/Controllers/AdminDuplicateAccountsController.cs @@ -2,6 +2,7 @@ using Microsoft.AspNetCore.Mvc; using Humans.Web.Authorization; using Humans.Web.Models; +using Humans.Application; using Humans.Application.Interfaces.Teams; using Humans.Application.Interfaces.Profiles; using Humans.Application.Interfaces.Users; @@ -63,16 +64,21 @@ public async Task Detail(Guid userId1, Guid userId2) var account1 = group.Accounts.First(a => a.UserId == userId1); var account2 = group.Accounts.First(a => a.UserId == userId2); - var profile1 = await BuildProfileCardAsync(userId1, account1); - var profile2 = await BuildProfileCardAsync(userId2, account2); + var info1 = await _userService.GetUserInfoAsync(userId1); + var info2 = await _userService.GetUserInfoAsync(userId2); + + var profile1 = await BuildProfileCardAsync(userId1, account1, info1); + var profile2 = await BuildProfileCardAsync(userId2, account2, info2); var viewModel = new DuplicateAccountDetailViewModel { SharedEmail = group.SharedEmail, Account1 = profile1, Account2 = profile2, - Account1EmailSources = account1.EmailSources, - Account2EmailSources = account2.EmailSources + Account1IdentityEmail = info1?.IdentityEmailColumn, + Account2IdentityEmail = info2?.IdentityEmailColumn, + Account1Emails = MapEmails(info1), + Account2Emails = MapEmails(info2) }; return View(viewModel); @@ -100,9 +106,9 @@ public async Task Resolve(Guid sourceUserId, Guid targetUserId, s } private async Task BuildProfileCardAsync( - Guid userId, DuplicateAccountInfo accountInfo) + Guid userId, DuplicateAccountInfo accountInfo, UserInfo? info) { - var profile = (await _userService.GetUserInfoAsync(userId))?.Profile; + var profile = info?.Profile; var teams = await teamService.GetUserTeamsAsync(userId); var activeTeamNames = teams .Where(m => m.LeftAt is null) @@ -125,4 +131,18 @@ private async Task BuildProfileCardAsync( Teams = activeTeamNames }; } + + private static List MapEmails(UserInfo? info) => + info is null + ? [] + : info.UserEmails + .Select(e => new DuplicateAccountEmailRowViewModel + { + Email = e.Email, + IsPrimary = e.IsPrimary, + IsVerified = e.IsVerified, + IsGoogle = e.IsGoogle, + Provider = e.Provider + }) + .ToList(); } diff --git a/src/Humans.Web/Controllers/Mailer/MailerAdminController.cs b/src/Humans.Web/Controllers/Mailer/MailerAdminController.cs index ed01843b0..45adca8ae 100644 --- a/src/Humans.Web/Controllers/Mailer/MailerAdminController.cs +++ b/src/Humans.Web/Controllers/Mailer/MailerAdminController.cs @@ -254,6 +254,7 @@ bool D(int prev, int now) || D(a.VerifiedFlipToOptIn, b.VerifiedFlipToOptIn) || D(a.VerifiedFlipToOptOut, b.VerifiedFlipToOptOut) || D(a.VerifiedKeepHumansPref, b.VerifiedKeepHumansPref) + || D(a.ResetMarketingFlag, b.ResetMarketingFlag) || D(a.AmbiguousMultipleVerified, b.AmbiguousMultipleVerified) || D(a.UnconfirmedSkipped, b.UnconfirmedSkipped); } diff --git a/src/Humans.Web/Extensions/Sections/MailerSectionExtensions.cs b/src/Humans.Web/Extensions/Sections/MailerSectionExtensions.cs index 061fb022b..176a67e50 100644 --- a/src/Humans.Web/Extensions/Sections/MailerSectionExtensions.cs +++ b/src/Humans.Web/Extensions/Sections/MailerSectionExtensions.cs @@ -52,6 +52,7 @@ internal static IServiceCollection AddMailerSection( services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddScoped(); services.AddTransient(); return services; diff --git a/src/Humans.Web/Models/AdminViewModels.cs b/src/Humans.Web/Models/AdminViewModels.cs index f96be7d48..4ccf7c978 100644 --- a/src/Humans.Web/Models/AdminViewModels.cs +++ b/src/Humans.Web/Models/AdminViewModels.cs @@ -358,8 +358,28 @@ public class DuplicateAccountDetailViewModel public string SharedEmail { get; set; } = string.Empty; public ProfileSummaryViewModel Account1 { get; set; } = new(); public ProfileSummaryViewModel Account2 { get; set; } = new(); - public List Account1EmailSources { get; set; } = []; - public List Account2EmailSources { get; set; } = []; + + /// Raw User.Email Identity column for account A (null when unset). + public string? Account1IdentityEmail { get; set; } + + /// Raw User.Email Identity column for account B (null when unset). + public string? Account2IdentityEmail { get; set; } + + /// All UserEmails rows for account A (full list, not just the conflicting overlap). + public List Account1Emails { get; set; } = []; + + /// All UserEmails rows for account B (full list, not just the conflicting overlap). + public List Account2Emails { get; set; } = []; +} + +/// One UserEmail row rendered on the duplicate-account detail page. +public class DuplicateAccountEmailRowViewModel +{ + public string Email { get; set; } = string.Empty; + public bool IsPrimary { get; set; } + public bool IsVerified { get; set; } + public bool IsGoogle { get; set; } + public string? Provider { get; set; } } /// diff --git a/src/Humans.Web/Views/AdminDuplicateAccounts/Detail.cshtml b/src/Humans.Web/Views/AdminDuplicateAccounts/Detail.cshtml index 8c443fb77..2a65bc97e 100644 --- a/src/Humans.Web/Views/AdminDuplicateAccounts/Detail.cshtml +++ b/src/Humans.Web/Views/AdminDuplicateAccounts/Detail.cshtml @@ -35,14 +35,56 @@
-
- Email sources: -
    - @foreach (var source in Model.Account1EmailSources) +
    +
    + User.Email (Identity column): + @if (!string.IsNullOrEmpty(Model.Account1IdentityEmail)) { -
  • @source
    • + @Model.Account1IdentityEmail } -
+ else + { + none + } +
+
+ UserEmails (@Model.Account1Emails.Count): + @if (Model.Account1Emails.Count == 0) + { + none + } + else + { +
    + @foreach (var email in Model.Account1Emails) + { +
  • + @email.Email + @if (email.IsPrimary) + { + Primary + } + @if (email.IsVerified) + { + Verified + } + else + { + Unverified + } + @if (email.IsGoogle) + { + Google + } + @if (!string.IsNullOrEmpty(email.Provider)) + { + @email.Provider + } +
    • + } +
+ } +
@@ -54,14 +96,56 @@
-
- Email sources: -
    - @foreach (var source in Model.Account2EmailSources) +
    +
    + User.Email (Identity column): + @if (!string.IsNullOrEmpty(Model.Account2IdentityEmail)) { -
  • @source
    • + @Model.Account2IdentityEmail } -
+ else + { + none + } +
+
+ UserEmails (@Model.Account2Emails.Count): + @if (Model.Account2Emails.Count == 0) + { + none + } + else + { +
    + @foreach (var email in Model.Account2Emails) + { +
  • + @email.Email + @if (email.IsPrimary) + { + Primary + } + @if (email.IsVerified) + { + Verified + } + else + { + Unverified + } + @if (email.IsGoogle) + { + Google + } + @if (!string.IsNullOrEmpty(email.Provider)) + { + @email.Provider + } +
    • + } +
+ } +
diff --git a/src/Humans.Web/Views/Mailer/Admin/Import.cshtml b/src/Humans.Web/Views/Mailer/Admin/Import.cshtml index 118891a7c..faf1d14d7 100644 --- a/src/Humans.Web/Views/Mailer/Admin/Import.cshtml +++ b/src/Humans.Web/Views/Mailer/Admin/Import.cshtml @@ -47,6 +47,10 @@ Human exists — keep Humans' pref (user touched recently) @Model.Plan.Counts.VerifiedKeepHumansPref.ToString("N0") + + Human exists — reset marketing flag (opt-in, not in Website group → null) + @Model.Plan.Counts.ResetMarketingFlag.ToString("N0") + Skipped — same email on multiple humans @Model.Plan.Counts.AmbiguousMultipleVerified.ToString("N0") @@ -73,6 +77,7 @@ (Outcome: SubscriberOutcome.VerifiedFlipToOptIn, Label: "Human Exists — Flip Marketing to Opt-In"), (Outcome: SubscriberOutcome.VerifiedFlipToOptOut, Label: "Human Exists — Flip Marketing to Opt-Out"), (Outcome: SubscriberOutcome.VerifiedKeepHumansPref, Label: "Human Exists — Keep Humans' Pref"), + (Outcome: SubscriberOutcome.ResetMarketingFlag, Label: "Human Exists — Reset Marketing Flag"), (Outcome: SubscriberOutcome.AmbiguousMultipleVerified, Label: "Skipped — Same Email on Multiple Humans"), (Outcome: SubscriberOutcome.UnconfirmedSkipped, Label: "Skipped — Unconfirmed in ML"), }; diff --git a/tests/Humans.Application.Tests/Services/Mailer/Audiences/MarketingNoTicketAudienceTests.cs b/tests/Humans.Application.Tests/Services/Mailer/Audiences/MarketingNoTicketAudienceTests.cs new file mode 100644 index 000000000..4b5f34ca8 --- /dev/null +++ b/tests/Humans.Application.Tests/Services/Mailer/Audiences/MarketingNoTicketAudienceTests.cs @@ -0,0 +1,86 @@ +using AwesomeAssertions; +using Humans.Application.Interfaces.Tickets; +using Humans.Application.Interfaces.Users; +using Humans.Application.Services.Mailer.Audiences; +using Humans.Application.Tests.Infrastructure; +using Humans.Domain.Entities; +using Humans.Domain.Enums; +using NodaTime; +using NSubstitute; + +namespace Humans.Application.Tests.Services.Mailer.Audiences; + +public class MarketingNoTicketAudienceTests +{ + [HumansFact] + public async Task ComputeMemberUserIdsAsync_ExcludesTicketHolders_AndNonOptIns() + { + var optInNoTicket = Guid.NewGuid(); // OptedOut=false, no ticket → IN + var optInWithTicket = Guid.NewGuid(); // OptedOut=false, has ticket → OUT + var optedOut = Guid.NewGuid(); // OptedOut=true → OUT + var noPrefRow = Guid.NewGuid(); // no row → OUT + + var audience = NewAudience( + users: new[] + { + UserWithMarketingPref(optInNoTicket, optedOut: false), + UserWithMarketingPref(optInWithTicket, optedOut: false), + UserWithMarketingPref(optedOut, optedOut: true), + UserWithoutMarketingPref(noPrefRow), + }, + ticketHolders: [optInWithTicket]); + + var members = await audience.ComputeMemberUserIdsAsync(CancellationToken.None); + + members.Should().BeEquivalentTo([optInNoTicket]); + } + + [HumansFact] + public async Task ComputeMemberUserIdsAsync_NoUsers_ReturnsEmpty() + { + var audience = NewAudience([], []); + + var members = await audience.ComputeMemberUserIdsAsync(CancellationToken.None); + + members.Should().BeEmpty(); + } + + [HumansFact] + public void Metadata_UsesHumansPrefix() + { + var audience = NewAudience([], []); + audience.Key.Should().Be("marketing-no-ticket"); + audience.MailerLiteGroupName.Should().Be("Humans - Marketing no Ticket"); + audience.MailerLiteGroupName.Should().StartWith("Humans - "); + } + + private static MarketingNoTicketAudience NewAudience( + IReadOnlyList users, + HashSet ticketHolders) + { + var userService = Substitute.For(); + userService.GetAllUserInfosAsync(Arg.Any()).Returns(users); + + var ticketService = Substitute.For(); + ticketService.GetUserIdsWithTicketsAsync().Returns(ticketHolders); + + return new MarketingNoTicketAudience(userService, ticketService); + } + + private static UserInfo UserWithMarketingPref(Guid userId, bool optedOut) => + UserInfo.Create( + new User { Id = userId, DisplayName = "u", PreferredLanguage = "en" }, + [], [], [], profile: null, [], [], [], + [new CommunicationPreference + { + Id = Guid.NewGuid(), + UserId = userId, + Category = MessageCategory.Marketing, + OptedOut = optedOut, + UpdatedAt = Instant.FromUnixTimeSeconds(0), + UpdateSource = "Test", + }]); + + private static UserInfo UserWithoutMarketingPref(Guid userId) => + UserInfoStubHelpers.MakeUserInfo(userId); +} diff --git a/tests/Humans.Application.Tests/Services/Mailer/ImportResultTests.cs b/tests/Humans.Application.Tests/Services/Mailer/ImportResultTests.cs index e2f33069a..2e036bf1b 100644 --- a/tests/Humans.Application.Tests/Services/Mailer/ImportResultTests.cs +++ b/tests/Humans.Application.Tests/Services/Mailer/ImportResultTests.cs @@ -12,6 +12,7 @@ private static ImportResult Sample( int prefsFlippedToOptIn = 2, int prefsFlippedToOptOut = 1, int prefsKeptByConflict = 1, + int marketingFlagsReset = 0, int unverifiedEmailsReplaced = 0, int ambiguousSkipped = 0, int unconfirmedSkipped = 5, @@ -25,6 +26,7 @@ private static ImportResult Sample( PrefsFlippedToOptIn: prefsFlippedToOptIn, PrefsFlippedToOptOut: prefsFlippedToOptOut, PrefsKeptByConflict: prefsKeptByConflict, + MarketingFlagsReset: marketingFlagsReset, UnverifiedEmailsReplaced: unverifiedEmailsReplaced, AmbiguousSkipped: ambiguousSkipped, UnconfirmedSkipped: unconfirmedSkipped, @@ -48,6 +50,7 @@ public void FormatSummary_IncludesAllKeyCounters() humansCreated: 3, prefsFlippedToOptIn: 2, prefsFlippedToOptOut: 1, + marketingFlagsReset: 6, unconfirmedSkipped: 5, vanishedBetweenPlanAndApply: 1, decisionsThrottled: 4, @@ -60,6 +63,7 @@ public void FormatSummary_IncludesAllKeyCounters() summary.Should().Contain("3 humans created"); summary.Should().Contain("2 flipped to opt-in"); summary.Should().Contain("1 flipped to opt-out"); + summary.Should().Contain("6 marketing flags reset"); summary.Should().Contain("5 unconfirmed skipped"); summary.Should().Contain("1 vanished"); summary.Should().Contain("4 throttled"); diff --git a/tests/Humans.Application.Tests/Services/Mailer/MailerImportServiceClassifierTests.cs b/tests/Humans.Application.Tests/Services/Mailer/MailerImportServiceClassifierTests.cs index dd96f3725..3c31f4c26 100644 --- a/tests/Humans.Application.Tests/Services/Mailer/MailerImportServiceClassifierTests.cs +++ b/tests/Humans.Application.Tests/Services/Mailer/MailerImportServiceClassifierTests.cs @@ -19,18 +19,18 @@ public class MailerImportServiceClassifierTests private static MailerLiteSubscriber Active(string email) => new("ml-id", email, "active", "api", Instant.FromUtc(2026, 1, 1, 0, 0), null, Instant.FromUtc(2026, 1, 1, 0, 0), - null, null, []); + null, null, [ClassifierHarness.WebsiteGroupId]); private static MailerLiteSubscriber Unsubscribed(string email, Instant? unsubscribedAt = null) => new("ml-id", email, "unsubscribed", "api", Instant.FromUtc(2026, 1, 1, 0, 0), unsubscribedAt ?? Instant.FromUtc(2026, 3, 1, 0, 0), Instant.FromUtc(2026, 1, 1, 0, 0), - null, null, []); + null, null, [ClassifierHarness.WebsiteGroupId]); private static MailerLiteSubscriber Unconfirmed(string email) => new("ml-id", email, "unconfirmed", "form", - null, null, null, null, null, []); + null, null, null, null, null, [ClassifierHarness.WebsiteGroupId]); [HumansFact] public async Task Classifies_UnconfirmedAsSkipped() @@ -207,6 +207,8 @@ public async Task Classifies_MultipleVerifiedOwnersAsAmbiguous() /// internal sealed class ClassifierHarness { + public const string WebsiteGroupId = "grp-website"; + private readonly IMailerLiteService _ml = Substitute.For(); private readonly IUserEmailService _userEmails = Substitute.For(); private readonly IUserService _users = Substitute.For(); @@ -279,6 +281,14 @@ public ClassifierHarness() .GetPreferenceOrNullAsync(Arg.Any(), Arg.Any(), Arg.Any()) .Returns(Task.FromResult(null)); + // Website group must resolve, and BuildPlanAsync's reset pass enumerates all users. + _ml.ListGroupsAsync(Arg.Any()) + .Returns(Task.FromResult>( + [new MailerLiteGroup(WebsiteGroupId, "Website", Instant.FromUtc(2020, 1, 1, 0, 0), 0, 0, 0, 0, 0)])); + _users + .GetAllUserInfosAsync(Arg.Any()) + .Returns(Task.FromResult>([])); + Service = new MailerImportService( _ml, _userEmails, diff --git a/tests/Humans.Application.Tests/Services/Mailer/MailerImportServiceConflictRuleTests.cs b/tests/Humans.Application.Tests/Services/Mailer/MailerImportServiceConflictRuleTests.cs index aad32307b..0d570c3a4 100644 --- a/tests/Humans.Application.Tests/Services/Mailer/MailerImportServiceConflictRuleTests.cs +++ b/tests/Humans.Application.Tests/Services/Mailer/MailerImportServiceConflictRuleTests.cs @@ -24,7 +24,7 @@ public async Task Apply_Bounce_AlwaysFlipsOptedOut() var bounced = new MailerLiteSubscriber( "ml-id", "user@x.com", "bounced", "api", - Instant.FromUtc(2026, 1, 1, 0, 0), null, null, null, null, []); + Instant.FromUtc(2026, 1, 1, 0, 0), null, null, null, null, [ApplyHarness.WebsiteGroupId]); harness.MlReturns(bounced); harness.SetVerifiedMatch("user@x.com", userId); @@ -50,7 +50,7 @@ public async Task Apply_UserActionAndNewer_PreservesHumansState() "ml-id", "user@x.com", "unsubscribed", "api", Instant.FromUtc(2026, 1, 1, 0, 0), Instant.FromUtc(2026, 4, 1, 0, 0), // UnsubscribedAt = 2026-04-01 (older than Humans) - null, null, null, []); + null, null, null, [ApplyHarness.WebsiteGroupId]); harness.MlReturns(unsubscribed); harness.SetVerifiedMatch("user@x.com", userId); @@ -76,7 +76,7 @@ public async Task Apply_SyncSource_OverwritesHumansState() "ml-id", "user@x.com", "unsubscribed", "api", Instant.FromUtc(2026, 1, 1, 0, 0), Instant.FromUtc(2026, 4, 1, 0, 0), // UnsubscribedAt = 2026-04-01 - null, null, null, []); + null, null, null, [ApplyHarness.WebsiteGroupId]); harness.MlReturns(unsubscribed); harness.SetVerifiedMatch("user@x.com", userId); @@ -106,6 +106,8 @@ private static ImportPlan MakePlan(string email, string status, SubscriberOutcom /// internal sealed class ApplyHarness { + public const string WebsiteGroupId = "grp-website"; + private readonly IMailerLiteService _ml = Substitute.For(); private readonly IUserEmailService _userEmails = Substitute.For(); private readonly ICommunicationPreferenceService _prefs = Substitute.For(); @@ -122,6 +124,11 @@ public ApplyHarness() Arg.Any(), Arg.Any()) .Returns(Task.CompletedTask); + // ApplyAsync resolves + filters by the Website group. + _ml.ListGroupsAsync(Arg.Any()) + .Returns(Task.FromResult>( + [new MailerLiteGroup(WebsiteGroupId, "Website", Instant.FromUtc(2020, 1, 1, 0, 0), 0, 0, 0, 0, 0)])); + Prefs = new PrefsVerifier(_prefs); Service = new MailerImportService( diff --git a/tests/Humans.Application.Tests/Services/Mailer/MailerImportServiceIdempotencyTests.cs b/tests/Humans.Application.Tests/Services/Mailer/MailerImportServiceIdempotencyTests.cs index 0f7a1681b..549bcd051 100644 --- a/tests/Humans.Application.Tests/Services/Mailer/MailerImportServiceIdempotencyTests.cs +++ b/tests/Humans.Application.Tests/Services/Mailer/MailerImportServiceIdempotencyTests.cs @@ -56,12 +56,13 @@ public async Task ApplyAsync_RunTwice_ProducesZeroPerRowEntries_AndOneSummary() internal sealed class IdempotencyHarness { private const string Email = "idempotent@x.com"; + private const string WebsiteGroupId = "grp-website"; // The ML subscriber is active (OptedOut=false when status="active"). private static readonly MailerLiteSubscriber ActiveSubscriber = new("ml-id", Email, "active", "api", Instant.FromUtc(2026, 1, 1, 0, 0), null, Instant.FromUtc(2026, 1, 1, 0, 0), - null, null, []); + null, null, [WebsiteGroupId]); private readonly Guid _userId = Guid.NewGuid(); @@ -69,6 +70,7 @@ internal sealed class IdempotencyHarness private readonly IUserEmailService _userEmails = Substitute.For(); private readonly IAccountProvisioningService _provisioning = Substitute.For(); private readonly ICommunicationPreferenceService _prefs = Substitute.For(); + private readonly IUserService _users = Substitute.For(); public AuditCounter Audit { get; } public MailerImportService Service { get; } @@ -79,6 +81,14 @@ public IdempotencyHarness() _ml.ListSubscribersAsync(Arg.Any()) .Returns(_ => new[] { ActiveSubscriber }.ToAsyncEnumerable()); + // Website group resolves; reset pass finds no candidate users. + _ml.ListGroupsAsync(Arg.Any()) + .Returns(Task.FromResult>( + [new MailerLiteGroup(WebsiteGroupId, "Website", Instant.FromUtc(2020, 1, 1, 0, 0), 0, 0, 0, 0, 0)])); + _users + .GetAllUserInfosAsync(Arg.Any()) + .Returns(Task.FromResult>([])); + // Pass 1: no verified human match → CreateContact path. _userEmails .GetDistinctVerifiedUserIdsAsync(Arg.Any(), Arg.Any()) @@ -106,7 +116,7 @@ public IdempotencyHarness() Service = new MailerImportService( _ml, _userEmails, - Substitute.For(), + _users, _provisioning, _prefs, Audit.Mock, diff --git a/tests/Humans.Application.Tests/Services/Mailer/MailerImportServiceThrottleTests.cs b/tests/Humans.Application.Tests/Services/Mailer/MailerImportServiceThrottleTests.cs index 864827299..828d8974f 100644 --- a/tests/Humans.Application.Tests/Services/Mailer/MailerImportServiceThrottleTests.cs +++ b/tests/Humans.Application.Tests/Services/Mailer/MailerImportServiceThrottleTests.cs @@ -141,6 +141,8 @@ public async Task ApplyAsync_NullLimit_ProcessesAll() internal sealed class ThrottleHarness { + public const string WebsiteGroupId = "grp-website"; + private readonly IMailerLiteService _ml = Substitute.For(); private readonly IUserEmailService _userEmails = Substitute.For(); private readonly IAccountProvisioningService _provisioning = Substitute.For(); @@ -164,6 +166,11 @@ public ThrottleHarness() .GetPreferenceOrNullAsync(Arg.Any(), Arg.Any(), Arg.Any()) .Returns(Task.FromResult(null)); + // ApplyAsync resolves + filters by the Website group. + _ml.ListGroupsAsync(Arg.Any()) + .Returns(Task.FromResult>( + [new MailerLiteGroup(WebsiteGroupId, "Website", Instant.FromUtc(2020, 1, 1, 0, 0), 0, 0, 0, 0, 0)])); + Service = new MailerImportService( _ml, _userEmails, Substitute.For(), _provisioning, _prefs, _audit, @@ -174,17 +181,17 @@ public ThrottleHarness() public static MailerLiteSubscriber Active(string email) => new("ml-id", email, "active", "api", Instant.FromUtc(2026, 1, 1, 0, 0), null, - Instant.FromUtc(2026, 1, 1, 0, 0), null, null, []); + Instant.FromUtc(2026, 1, 1, 0, 0), null, null, [WebsiteGroupId]); public static MailerLiteSubscriber Unsubscribed(string email) => new("ml-id", email, "unsubscribed", "api", Instant.FromUtc(2026, 1, 1, 0, 0), Instant.FromUtc(2026, 3, 1, 0, 0), - Instant.FromUtc(2026, 1, 1, 0, 0), null, null, []); + Instant.FromUtc(2026, 1, 1, 0, 0), null, null, [WebsiteGroupId]); public static MailerLiteSubscriber Unconfirmed(string email) => new("ml-id", email, "unconfirmed", "form", - null, null, null, null, null, []); + null, null, null, null, null, [WebsiteGroupId]); public void SetMlSubscribers(params MailerLiteSubscriber[] subscribers) { diff --git a/tests/Humans.Application.Tests/Services/Mailer/MailerImportServiceWebsiteScopeTests.cs b/tests/Humans.Application.Tests/Services/Mailer/MailerImportServiceWebsiteScopeTests.cs new file mode 100644 index 000000000..397598c67 --- /dev/null +++ b/tests/Humans.Application.Tests/Services/Mailer/MailerImportServiceWebsiteScopeTests.cs @@ -0,0 +1,281 @@ +using AwesomeAssertions; +using Humans.Application.Interfaces.AuditLog; +using Humans.Application.Interfaces.Mailer; +using Humans.Application.Interfaces.Mailer.Dtos; +using Humans.Application.Interfaces.Profiles; +using Humans.Application.Interfaces.Users; +using Humans.Application.Services.Mailer; +using Humans.Domain.Entities; +using Humans.Domain.Enums; +using Microsoft.Extensions.Logging.Abstractions; +using NodaTime; +using NodaTime.Testing; +using NSubstitute; + +namespace Humans.Application.Tests.Services.Mailer; + +/// +/// Covers the GDPR remediation: the import is scoped to the "Website" group, and +/// Marketing opt-ins the erroneous whole-account import left on people outside +/// that group are reset to null (row deleted), not opt-out. +/// +public class MailerImportServiceWebsiteScopeTests +{ + // After the 2026-05-19T00:01:01Z cutoff hardcoded in MailerImportService. + private static readonly Instant AfterCutoff = Instant.FromUtc(2026, 5, 19, 12, 0); + // Before the cutoff — represents genuine prior consent. + private static readonly Instant BeforeCutoff = Instant.FromUtc(2026, 5, 18, 0, 0); + + [HumansFact] + public async Task BuildPlan_ExcludesSubscribersOutsideWebsiteGroup() + { + var harness = new WebsiteScopeHarness(); + harness.SetSubscribers( + WebsiteScopeHarness.Active("in@x.com", inWebsite: true), + WebsiteScopeHarness.Active("out@x.com", inWebsite: false)); + + var plan = await harness.Service.BuildPlanAsync(); + + plan.TotalPulled.Should().Be(1); + plan.Decisions.Should().ContainSingle(d => d.Email == "in@x.com"); + plan.Decisions.Should().NotContain(d => d.Email == "out@x.com"); + } + + [HumansFact] + public async Task BuildPlan_AddsResetDecision_ForSyncOptInOutsideWebsite() + { + var harness = new WebsiteScopeHarness(); + harness.SetSubscribers(WebsiteScopeHarness.Active("member@x.com", inWebsite: true)); + + var ghostId = Guid.NewGuid(); + harness.SetUsers(Human(ghostId, "ghost@x.com", + optedOut: false, source: "MailerLiteSync", subscribedAt: AfterCutoff)); + + var plan = await harness.Service.BuildPlanAsync(); + + plan.Decisions.Should().ContainSingle(d => + d.Outcome == SubscriberOutcome.ResetMarketingFlag && d.TargetUserId == ghostId); + plan.Counts.ResetMarketingFlag.Should().Be(1); + } + + [HumansFact] + public async Task BuildPlan_NoReset_ForActiveWebsiteMember() + { + var harness = new WebsiteScopeHarness(); + harness.SetSubscribers(WebsiteScopeHarness.Active("member@x.com", inWebsite: true)); + + var id = Guid.NewGuid(); + harness.SetUsers(Human(id, "member@x.com", + optedOut: false, source: "MailerLiteSync", subscribedAt: AfterCutoff)); + + var plan = await harness.Service.BuildPlanAsync(); + + plan.Counts.ResetMarketingFlag.Should().Be(0); + } + + [HumansFact] + public async Task BuildPlan_NoReset_ForUnsubscribedWebsiteSubscriber() + { + // In the Website group but unsubscribed in ML. The import loop owns them + // (flips to opt-out, honouring the explicit unsubscribe), so the reset pass + // must NOT also null them — no double decision, no erasing an unsubscribe. + var harness = new WebsiteScopeHarness(); + harness.SetSubscribers(WebsiteScopeHarness.Unsubscribed("lapsed@x.com", inWebsite: true)); + + var id = Guid.NewGuid(); + harness.SetUsers(Human(id, "lapsed@x.com", + optedOut: false, source: "MailerLiteSync", subscribedAt: AfterCutoff)); + harness.MatchVerified("lapsed@x.com", id); + + var plan = await harness.Service.BuildPlanAsync(); + + plan.Counts.ResetMarketingFlag.Should().Be(0); + plan.Decisions.Should().NotContain(d => d.Outcome == SubscriberOutcome.ResetMarketingFlag); + // They're still handled by the import's verified-match path, not dropped. + plan.Decisions.Should().Contain(d => d.Email == "lapsed@x.com"); + } + + [HumansFact] + public async Task BuildPlan_NoReset_ForPriorConsent() + { + var harness = new WebsiteScopeHarness(); + harness.SetSubscribers(WebsiteScopeHarness.Active("member@x.com", inWebsite: true)); + + var id = Guid.NewGuid(); + harness.SetUsers(Human(id, "early@x.com", + optedOut: false, source: "MailerLiteSync", subscribedAt: BeforeCutoff)); + + var plan = await harness.Service.BuildPlanAsync(); + + plan.Counts.ResetMarketingFlag.Should().Be(0); + } + + [HumansFact] + public async Task BuildPlan_NoReset_ForUserSetOptIn() + { + var harness = new WebsiteScopeHarness(); + harness.SetSubscribers(WebsiteScopeHarness.Active("member@x.com", inWebsite: true)); + + var id = Guid.NewGuid(); + harness.SetUsers(Human(id, "self@x.com", + optedOut: false, source: "Profile", subscribedAt: AfterCutoff)); + + var plan = await harness.Service.BuildPlanAsync(); + + plan.Counts.ResetMarketingFlag.Should().Be(0); + } + + [HumansFact] + public async Task BuildPlan_NoReset_ForOptOut() + { + var harness = new WebsiteScopeHarness(); + harness.SetSubscribers(WebsiteScopeHarness.Active("member@x.com", inWebsite: true)); + + var id = Guid.NewGuid(); + harness.SetUsers(Human(id, "out@x.com", + optedOut: true, source: "MailerLiteSync", subscribedAt: null)); + + var plan = await harness.Service.BuildPlanAsync(); + + plan.Counts.ResetMarketingFlag.Should().Be(0); + } + + [HumansFact] + public async Task Apply_ResetDecision_DeletesPrefToNull() + { + var harness = new WebsiteScopeHarness(); + harness.SetSubscribers(WebsiteScopeHarness.Active("member@x.com", inWebsite: true)); + + var ghostId = Guid.NewGuid(); + harness.SetUserInfo(ghostId, Human(ghostId, "ghost@x.com", + optedOut: false, source: "MailerLiteSync", subscribedAt: AfterCutoff)); + + var plan = new ImportPlan( + [new SubscriberDecision("ghost@x.com", "n/a", SubscriberOutcome.ResetMarketingFlag, ghostId, null, null)], + TotalPulled: 1); + + var result = await harness.Service.ApplyAsync(plan, maxPerOutcome: null); + + result.MarketingFlagsReset.Should().Be(1); + await harness.Prefs.Received(1).ResetPreferenceAsync( + ghostId, MessageCategory.Marketing, "MailerLiteSyncReset", Arg.Any()); + } + + [HumansFact] + public async Task Apply_ResetDecision_SkippedWhenNoLongerCandidate() + { + // User flipped their own opt-in (source Profile) between preview and commit. + var harness = new WebsiteScopeHarness(); + harness.SetSubscribers(WebsiteScopeHarness.Active("member@x.com", inWebsite: true)); + + var id = Guid.NewGuid(); + harness.SetUserInfo(id, Human(id, "self@x.com", + optedOut: false, source: "Profile", subscribedAt: AfterCutoff)); + + var plan = new ImportPlan( + [new SubscriberDecision("self@x.com", "n/a", SubscriberOutcome.ResetMarketingFlag, id, null, null)], + TotalPulled: 1); + + var result = await harness.Service.ApplyAsync(plan, maxPerOutcome: null); + + result.MarketingFlagsReset.Should().Be(0); + await harness.Prefs.DidNotReceive().ResetPreferenceAsync( + Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()); + } + + [HumansFact] + public async Task BuildPlan_Throws_WhenWebsiteGroupMissing() + { + var harness = new WebsiteScopeHarness(includeWebsiteGroup: false); + harness.SetSubscribers(WebsiteScopeHarness.Active("anyone@x.com", inWebsite: false)); + + var act = async () => await harness.Service.BuildPlanAsync(); + + await act.Should().ThrowAsync() + .WithMessage("*Website*"); + } + + private static UserInfo Human( + Guid id, string email, bool optedOut, string source, Instant? subscribedAt) + { + var pref = new CommunicationPreference + { + Id = Guid.NewGuid(), + UserId = id, + Category = MessageCategory.Marketing, + OptedOut = optedOut, + UpdatedAt = Instant.FromUtc(2026, 5, 19, 12, 0), + UpdateSource = source, + SubscribedAt = subscribedAt, + }; + var ue = new UserEmail { Id = Guid.NewGuid(), UserId = id, Email = email, IsVerified = true, IsPrimary = true }; + return UserInfo.Create(new User { Id = id }, [ue], [], [], null, [], [], [], [pref]); + } +} + +internal sealed class WebsiteScopeHarness +{ + public const string WebsiteGroupId = "grp-website"; + private const string OtherGroupId = "grp-other"; + + private readonly IMailerLiteService _ml = Substitute.For(); + private readonly IUserEmailService _userEmails = Substitute.For(); + private readonly IUserService _users = Substitute.For(); + + public ICommunicationPreferenceService Prefs { get; } = Substitute.For(); + public MailerImportService Service { get; } + + public WebsiteScopeHarness(bool includeWebsiteGroup = true) + { + var groups = includeWebsiteGroup + ? new List { new(WebsiteGroupId, "Website", Instant.FromUtc(2020, 1, 1, 0, 0), 0, 0, 0, 0, 0) } + : []; + _ml.ListGroupsAsync(Arg.Any()) + .Returns(Task.FromResult>(groups)); + + // Default: no verified / unverified match → unmatched subscribers become CreateNewHuman. + _userEmails.GetDistinctVerifiedUserIdsAsync(Arg.Any(), Arg.Any()) + .Returns(Task.FromResult>([])); + _userEmails.FindAnyEmailRowByAddressAsync(Arg.Any(), Arg.Any()) + .Returns(Task.FromResult<(Guid, Guid)?>(null)); + + // Default: no reset-candidate users. + _users.GetAllUserInfosAsync(Arg.Any()) + .Returns(Task.FromResult>([])); + + Service = new MailerImportService( + _ml, _userEmails, _users, + Substitute.For(), + Prefs, + Substitute.For(), + new FakeClock(Instant.FromUtc(2026, 5, 20, 0, 0)), + NullLogger.Instance); + } + + public static MailerLiteSubscriber Active(string email, bool inWebsite) => + new("ml-" + email, email, "active", "api", + Instant.FromUtc(2026, 1, 1, 0, 0), null, Instant.FromUtc(2026, 1, 1, 0, 0), + null, null, [inWebsite ? WebsiteGroupId : OtherGroupId]); + + public static MailerLiteSubscriber Unsubscribed(string email, bool inWebsite) => + new("ml-" + email, email, "unsubscribed", "api", + Instant.FromUtc(2026, 1, 1, 0, 0), Instant.FromUtc(2026, 3, 1, 0, 0), + Instant.FromUtc(2026, 1, 1, 0, 0), + null, null, [inWebsite ? WebsiteGroupId : OtherGroupId]); + + public void SetSubscribers(params MailerLiteSubscriber[] subscribers) => + _ml.ListSubscribersAsync(Arg.Any()) + .Returns(_ => subscribers.ToAsyncEnumerable()); + + public void SetUsers(params UserInfo[] users) => + _users.GetAllUserInfosAsync(Arg.Any()) + .Returns(Task.FromResult>(users)); + + public void MatchVerified(string email, Guid userId) => + _userEmails.GetDistinctVerifiedUserIdsAsync(email, Arg.Any()) + .Returns(Task.FromResult>([userId])); + + public void SetUserInfo(Guid id, UserInfo info) => + _users.GetUserInfoAsync(id, Arg.Any()) + .Returns(new ValueTask(info)); +} diff --git a/tests/Humans.Web.Tests/Controllers/Mailer/MailerAdminControllerTests.cs b/tests/Humans.Web.Tests/Controllers/Mailer/MailerAdminControllerTests.cs index 772242f76..b8336ef66 100644 --- a/tests/Humans.Web.Tests/Controllers/Mailer/MailerAdminControllerTests.cs +++ b/tests/Humans.Web.Tests/Controllers/Mailer/MailerAdminControllerTests.cs @@ -78,7 +78,8 @@ private static SubscriberDecision Decision(SubscriberOutcome outcome) => private static ImportResult StubResult() => new(TotalPulled: 10, HumansCreated: 2, PrefsFlippedToOptIn: 2, PrefsFlippedToOptOut: 1, - PrefsKeptByConflict: 0, UnverifiedEmailsReplaced: 0, + PrefsKeptByConflict: 0, MarketingFlagsReset: 0, + UnverifiedEmailsReplaced: 0, AmbiguousSkipped: 0, UnconfirmedSkipped: 0, VanishedBetweenPlanAndApply: 0, DecisionsThrottled: 0, Errors: 0, Elapsed: Duration.Zero); @@ -98,6 +99,7 @@ public async Task Commit_RedirectsToPreview_WhenCountsDriftedMoreThan10Percent() VerifiedFlipToOptIn: 0, VerifiedFlipToOptOut: 0, VerifiedKeepHumansPref: 0, + ResetMarketingFlag: 0, AmbiguousMultipleVerified: 0, UnconfirmedSkipped: 0); @@ -303,6 +305,7 @@ public async Task Commit_ExecutesApply_WhenCountsWithinTolerance() VerifiedFlipToOptIn: 5, VerifiedFlipToOptOut: 0, VerifiedKeepHumansPref: 0, + ResetMarketingFlag: 0, AmbiguousMultipleVerified: 0, UnconfirmedSkipped: 0); diff --git a/tests/e2e/package-lock.json b/tests/e2e/package-lock.json index 298a0ac9a..e0589c4fe 100644 --- a/tests/e2e/package-lock.json +++ b/tests/e2e/package-lock.json @@ -6,18 +6,18 @@ "": { "name": "humans-e2e", "devDependencies": { - "@playwright/test": "^1.52.0", - "typescript": "^5.8.0" + "@playwright/test": "^1.60.0", + "typescript": "^6.0.3" } }, "node_modules/@playwright/test": { - "version": "1.58.2", - "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.58.2.tgz", - "integrity": "sha512-akea+6bHYBBfA9uQqSYmlJXn61cTa+jbO87xVLCWbTqbWadRVmhxlXATaOjOgcBaWU4ePo0wB41KMFv3o35IXA==", + "version": "1.60.0", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.60.0.tgz", + "integrity": "sha512-O71yZIbAh/PxDMNGns37GHBIfrVkEVyn+AXyIa5dOTfb4/xNvRWV+Vv/NMbNCtODB/pO7vLlF2OTmMVLhmr7Ag==", "dev": true, "license": "Apache-2.0", "dependencies": { - "playwright": "1.58.2" + "playwright": "1.60.0" }, "bin": { "playwright": "cli.js" @@ -42,13 +42,13 @@ } }, "node_modules/playwright": { - "version": "1.58.2", - "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.58.2.tgz", - "integrity": "sha512-vA30H8Nvkq/cPBnNw4Q8TWz1EJyqgpuinBcHET0YVJVFldr8JDNiU9LaWAE1KqSkRYazuaBhTpB5ZzShOezQ6A==", + "version": "1.60.0", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.60.0.tgz", + "integrity": "sha512-hheHdokM8cdqCb0lcE3s+zT4t4W+vvjpGxsZlDnikarzx8tSzMebh3UiFtgqwFwnTnjYQcsyMF8ei2mCO/tpeA==", "dev": true, "license": "Apache-2.0", "dependencies": { - "playwright-core": "1.58.2" + "playwright-core": "1.60.0" }, "bin": { "playwright": "cli.js" @@ -61,9 +61,9 @@ } }, "node_modules/playwright-core": { - "version": "1.58.2", - "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.58.2.tgz", - "integrity": "sha512-yZkEtftgwS8CsfYo7nm0KE8jsvm6i/PTgVtB8DL726wNf6H2IMsDuxCpJj59KDaxCtSnrWan2AeDqM7JBaultg==", + "version": "1.60.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.60.0.tgz", + "integrity": "sha512-9bW6zvX/m0lEbgTKJ6YppOKx8H3VOPBMOCFh2irXFOT4BbHgrx5hPjwJYLT40Lu+4qtD36qKc/Hn56StUW57IA==", "dev": true, "license": "Apache-2.0", "bin": { @@ -74,9 +74,9 @@ } }, "node_modules/typescript": { - "version": "5.9.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", - "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-6.0.3.tgz", + "integrity": "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==", "dev": true, "license": "Apache-2.0", "bin": { diff --git a/tests/e2e/package.json b/tests/e2e/package.json index 17a25bc87..85835ccc6 100644 --- a/tests/e2e/package.json +++ b/tests/e2e/package.json @@ -7,7 +7,7 @@ "report": "playwright show-report" }, "devDependencies": { - "@playwright/test": "^1.52.0", - "typescript": "^5.8.0" + "@playwright/test": "^1.60.0", + "typescript": "^6.0.3" } }