From de7ba89a7d29c55add5dc54a207511736d38d556 Mon Sep 17 00:00:00 2001 From: Anil Kulkarni Date: Thu, 17 Sep 2026 16:44:36 -0700 Subject: [PATCH 1/3] Add the user id to the slack message --- .../coding-plans/minimax-token-health.test.ts | 59 ++++++++++++++++++- .../lib/coding-plans/minimax-token-health.ts | 28 ++++++++- 2 files changed, 85 insertions(+), 2 deletions(-) diff --git a/apps/web/src/lib/coding-plans/minimax-token-health.test.ts b/apps/web/src/lib/coding-plans/minimax-token-health.test.ts index c94531f958..3f42426350 100644 --- a/apps/web/src/lib/coding-plans/minimax-token-health.test.ts +++ b/apps/web/src/lib/coding-plans/minimax-token-health.test.ts @@ -2,6 +2,10 @@ jest.mock('@/lib/ai-gateway/byok/encryption', () => ({ decryptApiKey: jest.fn(() => 'plaintext-api-key'), })); +jest.mock('@/lib/utils.server', () => ({ + sentryLogger: jest.fn(() => jest.fn()), +})); + const mockOrderBy = jest.fn, []>(); jest.mock('@/lib/drizzle', () => ({ db: { @@ -17,7 +21,8 @@ jest.mock('@/lib/drizzle', () => ({ }, })); -import { afterEach, describe, expect, it } from '@jest/globals'; +import { afterEach, beforeAll, describe, expect, it } from '@jest/globals'; +import { sentryLogger } from '@/lib/utils.server'; import type { AdminSlackNotification } from '@/lib/slack/admin-notifications'; import { @@ -31,6 +36,12 @@ import { type MiniMaxTokenHealthTarget, } from './minimax-token-health'; +// sentryLogger('minimax-token-health', 'info') runs once when +// minimax-token-health.ts loads (see `logBadResponse`). Module loading +// finishes before any test lifecycle hook runs, so by `beforeAll` the mocked +// sentryLogger's first call result is the logger instance under test. +let mockLogBadResponse: jest.Mock; + function target(overrides: Partial = {}): MiniMaxTokenHealthTarget { return { inventoryId: 'inv-1', @@ -58,8 +69,13 @@ function entry(overrides: Partial = {}): MiniMaxTokenHe }; } +beforeAll(() => { + mockLogBadResponse = jest.mocked(sentryLogger).mock.results[0]!.value as jest.Mock; +}); + afterEach(() => { jest.restoreAllMocks(); + mockLogBadResponse.mockClear(); }); describe('probeMiniMaxTokenPlanRemains', () => { @@ -229,6 +245,7 @@ describe('buildMiniMaxTokenHealthSlackNotification', () => { inventoryId: 'inv-2', subscriptionId: 'sub-2', upstreamPlanId: '522937595127087110', + userId: 'user-2', category: 'bad_response', reason: 'provider_plan_inactive', subscriptionStatus: 'active', @@ -237,6 +254,7 @@ describe('buildMiniMaxTokenHealthSlackNotification', () => { inventoryId: 'inv-3', subscriptionId: 'sub-3', upstreamPlanId: '522937971515547655', + userId: 'user-3', category: 'denied', reason: 'http_429', httpStatus: 429, @@ -256,8 +274,10 @@ describe('buildMiniMaxTokenHealthSlackNotification', () => { const rendered = JSON.stringify(result.notification.blocks); expect(rendered).toContain('522937595127087110'); expect(rendered).toContain('provider_plan_inactive'); + expect(rendered).toContain('user-2'); expect(rendered).toContain('522937971515547655'); expect(rendered).toContain('http_429 (HTTP 429)'); + expect(rendered).toContain('user-3'); expect(rendered).not.toContain('inv-4'); expect(result.notification.text).toContain('`2` active subscriptions need follow-up'); }); @@ -295,4 +315,41 @@ describe('sendMiniMaxTokenHealthSlackSummary', () => { expect.objectContaining({ text: expect.stringContaining('`1` checked') }) ); }); + + it('logs full detail for each bad_response entry and skips other categories', async () => { + const checkAll = jest.fn(async () => [ + entry({ inventoryId: 'inv-1', category: 'healthy', reason: 'ok' }), + entry({ + inventoryId: 'inv-2', + planId: 'minimax-token-plan-plus', + upstreamPlanId: '522935352948625416', + subscriptionId: 'sub-2', + userId: 'user-2', + subscriptionStatus: 'active', + category: 'bad_response', + reason: 'invalid_response', + }), + entry({ + inventoryId: 'inv-3', + category: 'denied', + reason: 'http_401', + httpStatus: 401, + }), + ]); + const sendNotification = jest.fn(async (_notification: AdminSlackNotification) => undefined); + + await sendMiniMaxTokenHealthSlackSummary({ checkAll, sendNotification }); + + expect(mockLogBadResponse).toHaveBeenCalledTimes(1); + expect(mockLogBadResponse).toHaveBeenCalledWith('MiniMax token health bad_response', { + inventoryId: 'inv-2', + planId: 'minimax-token-plan-plus', + upstreamPlanId: '522935352948625416', + subscriptionId: 'sub-2', + userId: 'user-2', + subscriptionStatus: 'active', + reason: 'invalid_response', + httpStatus: undefined, + }); + }); }); diff --git a/apps/web/src/lib/coding-plans/minimax-token-health.ts b/apps/web/src/lib/coding-plans/minimax-token-health.ts index 00dce1bcf7..617d6e5695 100644 --- a/apps/web/src/lib/coding-plans/minimax-token-health.ts +++ b/apps/web/src/lib/coding-plans/minimax-token-health.ts @@ -13,12 +13,15 @@ import { sendAdminSlackNotification, type AdminSlackNotification, } from '@/lib/slack/admin-notifications'; +import { sentryLogger } from '@/lib/utils.server'; import { coding_plan_key_inventory, coding_plan_subscriptions } from '@kilocode/db/schema'; import { CodingPlanSubscriptionStatus } from '@kilocode/db/schema-types'; import type { EncryptedData } from '@kilocode/db/schema-types'; const MINIMAX_HEALTH_CHECK_TIMEOUT_MS = 8_000; +const logBadResponse = sentryLogger('minimax-token-health', 'info'); + // MiniMax has been observed to apply dynamic rate limiting under load // (see the 2026-09-15 Token Plan incident), so this sweep bounds its own // fan-out to avoid generating a burst of load that produces its own denials. @@ -303,7 +306,8 @@ function followUpLine(entry: MiniMaxTokenHealthEntry): string { const httpStatus = entry.httpStatus ? ` (HTTP ${entry.httpStatus})` : ''; const upstreamPlanId = escapeSlackLabel(entry.upstreamPlanId); const subscriptionId = escapeSlackLabel(entry.subscriptionId ?? 'n/a'); - return `${planDisplayName(entry.planId)} · upstream \`${upstreamPlanId}\` · sub \`${subscriptionId}\` (${statusLabel}) · *${entry.category}*: ${entry.reason}${httpStatus}`; + const userId = escapeSlackLabel(entry.userId ?? 'n/a'); + return `${planDisplayName(entry.planId)} · upstream \`${upstreamPlanId}\` · sub \`${subscriptionId}\` (${statusLabel}) · user \`${userId}\` · *${entry.category}*: ${entry.reason}${httpStatus}`; } export function buildMiniMaxTokenHealthSlackNotification( @@ -379,6 +383,27 @@ export function buildMiniMaxTokenHealthSlackNotification( }; } +// Slack truncates the follow-up list to DETAIL_LIMIT and only lists entries +// with a live subscription, so this logs every `bad_response` entry (the +// category that can't be told apart from a denied/unreachable credential by +// HTTP status alone) with its raw reason so the underlying cause is visible +// in cron logs even when Slack omits or aggregates it. +function logBadResponseEntries(entries: MiniMaxTokenHealthEntry[]): void { + for (const entry of entries) { + if (entry.category !== 'bad_response') continue; + logBadResponse('MiniMax token health bad_response', { + inventoryId: entry.inventoryId, + planId: entry.planId, + upstreamPlanId: entry.upstreamPlanId, + subscriptionId: entry.subscriptionId, + userId: entry.userId, + subscriptionStatus: entry.subscriptionStatus, + reason: entry.reason, + httpStatus: entry.httpStatus, + }); + } +} + type SendSummaryDependencies = { checkAll?: typeof checkAllMiniMaxTokenHealth; sendNotification?: typeof sendAdminSlackNotification; @@ -389,6 +414,7 @@ export async function sendMiniMaxTokenHealthSlackSummary({ sendNotification = sendAdminSlackNotification, }: SendSummaryDependencies = {}): Promise { const entries = await checkAll(); + logBadResponseEntries(entries); const { notification, totals } = buildMiniMaxTokenHealthSlackNotification(entries); await sendNotification(notification); return totals; From 238de44f71968f85020e14ee16c1c6fb591b83db Mon Sep 17 00:00:00 2001 From: Anil Kulkarni Date: Thu, 17 Sep 2026 18:09:55 -0700 Subject: [PATCH 2/3] Add swap button and reduce inventory button --- .../CodingPlansOperationsContent.tsx | 279 +++++++++++++++++- .../coding-plan-operations.test.ts | 53 ++++ .../coding-plans/coding-plan-operations.ts | 49 +++ apps/web/src/lib/coding-plans/index.ts | 254 ++++++++++++++-- .../src/routers/coding-plans-router.test.ts | 101 +++++++ apps/web/src/routers/coding-plans-router.ts | 43 +++ 6 files changed, 746 insertions(+), 33 deletions(-) diff --git a/apps/web/src/app/admin/coding-plans/CodingPlansOperationsContent.tsx b/apps/web/src/app/admin/coding-plans/CodingPlansOperationsContent.tsx index 25634e4bb8..a3553dc6ab 100644 --- a/apps/web/src/app/admin/coding-plans/CodingPlansOperationsContent.tsx +++ b/apps/web/src/app/admin/coding-plans/CodingPlansOperationsContent.tsx @@ -41,6 +41,7 @@ import { Textarea } from '@/components/ui/textarea'; import { SubscriptionStatusBadge } from '@/components/subscriptions/SubscriptionStatusBadge'; import { canSubmitExtensionDays, + canSubmitReduceCount, getCancelSubscriptionDialogCopy, getCodingPlanInsights, getCodingPlanProviderDisplayName, @@ -48,11 +49,15 @@ import { getInventoryReplacementCompleteToast, getInventoryReplacementDialogCopy, getPlanPerformanceRows, + getReduceInventoryCompleteToast, + getReduceInventoryDialogCopy, getReplacementCompleteToast, getReplacementDialogCopy, getRevocationCompleteToast, getRevocationDialogCopy, getSubscriptionSummaryItems, + getSwapCredentialCompleteToast, + getSwapCredentialDialogCopy, type InsightsRangeDays, } from '@/app/admin/coding-plans/coding-plan-operations'; import { @@ -81,6 +86,9 @@ type OperationsState = { cancelSelection: AdminCodingPlanSubscriptionItem | null; extendSelection: AdminCodingPlanSubscriptionItem | null; extendDays: string; + swapSelection: AdminCodingPlanSubscriptionItem | null; + reduceSelection: ReduceInventorySelection | null; + reduceCount: string; }; const EMPTY_SUBSCRIPTION_SUMMARY = { @@ -120,6 +128,9 @@ const INITIAL_OPERATIONS_STATE: OperationsState = { cancelSelection: null, extendSelection: null, extendDays: '7', + swapSelection: null, + reduceSelection: null, + reduceCount: '1', }; function updateOperationsState(state: OperationsState, update: Partial) { @@ -143,6 +154,9 @@ export function CodingPlansOperationsContent() { cancelSelection, extendSelection, extendDays, + swapSelection, + reduceSelection, + reduceCount, } = state; const [subscriptionPage, setSubscriptionPage] = useState(1); const [subscriptionSearchDraft, setSubscriptionSearchDraft] = useState(''); @@ -169,6 +183,11 @@ export function CodingPlansOperationsContent() { const setExtendSelection = (extendSelection: AdminCodingPlanSubscriptionItem | null) => updateState({ extendSelection }); const setExtendDays = (extendDays: string) => updateState({ extendDays }); + const setSwapSelection = (swapSelection: AdminCodingPlanSubscriptionItem | null) => + updateState({ swapSelection }); + const setReduceSelection = (reduceSelection: ReduceInventorySelection | null) => + updateState({ reduceSelection }); + const setReduceCount = (reduceCount: string) => updateState({ reduceCount }); const countsQuery = useQuery(trpc.codingPlans.adminKeyInventory.queryOptions({})); const queueQuery = useQuery(trpc.codingPlans.adminRevocationQueue.queryOptions({})); @@ -268,11 +287,41 @@ export function CodingPlansOperationsContent() { onError: error => toast.error(error.message || 'Unable to extend subscription.'), }) ); + const swapMutation = useMutation( + trpc.codingPlans.adminReassignSubscriptionCredential.mutationOptions({ + onSuccess: async () => { + setSwapSelection(null); + toast.success(getSwapCredentialCompleteToast()); + await refreshOperations(); + }, + onError: error => toast.error(error.message || 'Unable to swap the subscription credential.'), + }) + ); + const reduceMutation = useMutation( + trpc.codingPlans.adminReduceInventory.mutationOptions({ + onSuccess: async (result, variables) => { + setReduceSelection(null); + setReduceCount('1'); + toast.success( + getReduceInventoryCompleteToast( + result.queued.length, + variables.count, + result.queued.map(item => item.upstreamPlanId) + ) + ); + await refreshOperations(); + }, + onError: error => toast.error(error.message || 'Unable to queue inventory for removal.'), + }) + ); const submittedEntries = entriesText .split('\n') .map(entry => entry.trim()) .filter(entry => entry.length > 0); const canSubmitExtend = canSubmitExtensionDays(extendDays); + const canSubmitReduce = reduceSelection + ? canSubmitReduceCount(reduceCount, reduceSelection.availableCount) + : false; useEffect(() => { const normalizedPage = subscriptionsQuery.data?.pagination.page; @@ -313,6 +362,11 @@ export function CodingPlansOperationsContent() { const totalCredentialCount = inventoryCounts.reduce((total, item) => total + item.count, 0); const countCredentialsByStatus = (status: string) => inventoryCounts.reduce((total, item) => total + (item.status === status ? item.count : 0), 0); + const availableCountsByPlan = inventoryCounts.reduce>((totals, item) => { + if (item.status !== 'available') return totals; + totals[item.planId] = (totals[item.planId] ?? 0) + item.count; + return totals; + }, {}); const inventorySummary = [ { label: 'Total credentials in system', @@ -433,6 +487,8 @@ export function CodingPlansOperationsContent() { setExtendSelection(item); setExtendDays('7'); }} + onSwap={setSwapSelection} + availableCountsByPlan={availableCountsByPlan} /> @@ -487,6 +543,17 @@ export function CodingPlansOperationsContent() { entries: submittedEntries, }); }} + onReduce={row => + setReduceSelection({ + providerId: row.providerId, + planId: row.planId, + providerDisplayName: getCodingPlanProviderDisplayName( + providerOptions, + row.providerId + ), + availableCount: row.statusCounts.available ?? 0, + }) + } inventoryReplacementId={inventoryReplacementId} inventoryReplacementApiKey={inventoryReplacementApiKey} onInventoryReplacementIdChange={setInventoryReplacementId} @@ -534,6 +601,22 @@ export function CodingPlansOperationsContent() { }} onExtendDaysChange={setExtendDays} onExtend={(subscriptionId, days) => extendMutation.mutate({ subscriptionId, days })} + swapSelection={swapSelection} + swapPending={swapMutation.isPending} + onCloseSwap={() => setSwapSelection(null)} + onSwap={subscriptionId => swapMutation.mutate({ subscriptionId })} + reduceSelection={reduceSelection} + reduceCount={reduceCount} + reducePending={reduceMutation.isPending} + canSubmitReduce={canSubmitReduce} + onCloseReduce={() => { + setReduceSelection(null); + setReduceCount('1'); + }} + onReduceCountChange={setReduceCount} + onReduce={(planId, count) => + reduceMutation.mutate({ planId: planId as CodingPlanId, count }) + } inventoryReplacementId={inventoryReplacementId} inventoryReplacementConfirmOpen={inventoryReplacementConfirmOpen} inventoryReplacementPending={inventoryReplacementMutation.isPending} @@ -643,6 +726,16 @@ type RevocationSelection = { providerDisplayName: string; }; +// The Inventory counts row selected for a "Reduce inventory" action. Kept +// whole (rather than just an ID) so the dialog can show the provider name and +// cap the requested count at what is actually available for that plan. +type ReduceInventorySelection = { + providerId: string; + planId: string; + providerDisplayName: string; + availableCount: number; +}; + function SummaryCards({ items, isLoading, @@ -842,14 +935,16 @@ function InventoryCountsTable({ items, isLoading, isError, + onReduce, }: { items: InventoryCountItem[]; isLoading: boolean; isError: boolean; + onReduce: (row: InventoryCountRow) => void; }) { const statusColumns = getInventoryStatusColumns(items); const rows = getInventoryCountRows(items); - const columnCount = 3 + statusColumns.length; + const columnCount = 4 + statusColumns.length; const metricColumnKeys = ['loaded', ...statusColumns]; return ( @@ -860,13 +955,14 @@ function InventoryCountsTable({
- +
{metricColumnKeys.map(key => ( ))} + @@ -878,6 +974,7 @@ function InventoryCountsTable({ {formatStatusTitle(status)} ))} + Actions @@ -897,20 +994,37 @@ function InventoryCountsTable({ ) : ( - rows.map(row => ( - - {row.providerId} - {row.planId} - - {row.loadedCount} - - {statusColumns.map(status => ( - - {row.statusCounts[status] ?? 0} + rows.map(row => { + const availableCount = row.statusCounts.available ?? 0; + return ( + + {row.providerId} + {row.planId} + + {row.loadedCount} - ))} - - )) + {statusColumns.map(status => ( + + {row.statusCounts[status] ?? 0} + + ))} + + + + + ); + }) )}
@@ -997,6 +1111,8 @@ function SubscriptionsTable({ onPageChange, onCancel, onExtend, + onSwap, + availableCountsByPlan, }: { items: AdminCodingPlanSubscriptionItem[]; pagination?: { page: number; total: number; totalPages: number }; @@ -1012,6 +1128,8 @@ function SubscriptionsTable({ onPageChange: (page: number) => void; onCancel: (item: AdminCodingPlanSubscriptionItem) => void; onExtend: (item: AdminCodingPlanSubscriptionItem) => void; + onSwap: (item: AdminCodingPlanSubscriptionItem) => void; + availableCountsByPlan: Record; }) { const currentPage = pagination?.page ?? 1; const totalPages = Math.max(pagination?.totalPages ?? 1, 1); @@ -1126,6 +1244,10 @@ function SubscriptionsTable({ const canExtend = item.status === 'active'; const inventoryKeyId = item.inventoryKeyId; const upstreamPlanId = item.upstreamPlanId; + const canSwap = + inventoryKeyId !== null && + (item.status === 'active' || item.status === 'past_due'); + const availableForPlan = availableCountsByPlan[item.planId] ?? 0; return ( @@ -1212,6 +1334,21 @@ function SubscriptionsTable({
+ {canSwap ? ( + + ) : null} {canExtend ? ( + + + + + + { + if (!open && !reducePending) onCloseReduce(); + }} + > + + + {reduceCopy.title} + {reduceCopy.description} + +
+ + onReduceCountChange(event.target.value)} + disabled={reducePending} + /> + {reduceSelection ? ( +

+ {reduceSelection.availableCount} available for {reduceSelection.providerDisplayName}{' '} + {reduceSelection.planId}. +

+ ) : null} +
+ + + + + + + { diff --git a/apps/web/src/app/admin/coding-plans/coding-plan-operations.test.ts b/apps/web/src/app/admin/coding-plans/coding-plan-operations.test.ts index 18389ce474..74067130bd 100644 --- a/apps/web/src/app/admin/coding-plans/coding-plan-operations.test.ts +++ b/apps/web/src/app/admin/coding-plans/coding-plan-operations.test.ts @@ -1,5 +1,6 @@ import { canSubmitExtensionDays, + canSubmitReduceCount, getCancelSubscriptionDialogCopy, getCodingPlanInsights, getCodingPlanProviderDisplayName, @@ -7,11 +8,15 @@ import { getInventoryReplacementCompleteToast, getInventoryReplacementDialogCopy, getPlanPerformanceRows, + getReduceInventoryCompleteToast, + getReduceInventoryDialogCopy, getReplacementCompleteToast, getReplacementDialogCopy, getRevocationCompleteToast, getRevocationDialogCopy, getSubscriptionSummaryItems, + getSwapCredentialCompleteToast, + getSwapCredentialDialogCopy, } from '@/app/admin/coding-plans/coding-plan-operations'; const CATALOG = [ @@ -107,6 +112,54 @@ describe('subscription action copy', () => { 'without charging credits' ); }); + + it('names the user in the swap credential dialog and warns billing is unaffected', () => { + expect(getSwapCredentialDialogCopy('Ada').title).toBe("Swap Ada's credential?"); + expect(getSwapCredentialDialogCopy('Ada').description).toContain('Billing'); + expect(getSwapCredentialCompleteToast()).toBe('Subscription moved to a new pooled credential.'); + }); +}); + +describe('reduce inventory copy', () => { + it('names the plan and available count, and explains the manual provider-side step', () => { + const dialog = getReduceInventoryDialogCopy('minimax-token-plan-plus', 5); + expect(dialog.title).toBe('Queue minimax-token-plan-plus inventory for removal?'); + expect(dialog.description).toContain('5 credentials are currently available'); + expect(dialog.description).toContain('Pending Key Rotation'); + expect(dialog.description).toContain('Kilo never contacts the provider'); + }); + + it('uses singular phrasing for a single available credential', () => { + const dialog = getReduceInventoryDialogCopy('minimax-token-plan-plus', 1); + expect(dialog.description).toContain('1 credential is currently available'); + }); + + it('bounds the requested count to a positive integer no greater than what is available', () => { + expect(canSubmitReduceCount('3', 5)).toBe(true); + expect(canSubmitReduceCount('5', 5)).toBe(true); + expect(canSubmitReduceCount('6', 5)).toBe(false); + expect(canSubmitReduceCount('0', 5)).toBe(false); + expect(canSubmitReduceCount('1.5', 5)).toBe(false); + expect(canSubmitReduceCount('abc', 5)).toBe(false); + }); + + it('reports the exact queued count and upstream plan IDs on success', () => { + expect(getReduceInventoryCompleteToast(3, 3, ['plan-a', 'plan-b', 'plan-c'])).toBe( + 'Queued 3 credentials for removal in Pending Key Rotation. Deprovision using upstream plan IDs: plan-a, plan-b, plan-c' + ); + }); + + it('flags a shortfall when fewer credentials were queued than requested', () => { + expect(getReduceInventoryCompleteToast(2, 5, ['plan-a', 'plan-b'])).toBe( + 'Queued 2 of 5 requested credentials for removal (the rest were claimed before queuing). Find them in Pending Key Rotation and deprovision using upstream plan IDs: plan-a, plan-b' + ); + }); + + it('explains that nothing was queued when every candidate was already claimed', () => { + expect(getReduceInventoryCompleteToast(0, 2, [])).toBe( + 'No credentials were queued. They may have just been claimed by a subscription; refresh and try again.' + ); + }); }); describe('aggregate insight helpers', () => { diff --git a/apps/web/src/app/admin/coding-plans/coding-plan-operations.ts b/apps/web/src/app/admin/coding-plans/coding-plan-operations.ts index 13de7115e6..6327f4cfcb 100644 --- a/apps/web/src/app/admin/coding-plans/coding-plan-operations.ts +++ b/apps/web/src/app/admin/coding-plans/coding-plan-operations.ts @@ -92,6 +92,55 @@ export function getExtendSubscriptionDialogCopy(userName: string): { }; } +export function getSwapCredentialDialogCopy(userName: string): { + title: string; + description: string; +} { + return { + title: `Swap ${userName}'s credential?`, + description: + "Moves this subscription onto the next available credential from the pool and queues the current one for manual revocation. Billing, renewal dates, and status are unaffected, and the user doesn't need to do anything.", + }; +} + +export function getSwapCredentialCompleteToast(): string { + return 'Subscription moved to a new pooled credential.'; +} + +export function getReduceInventoryDialogCopy( + planId: string, + availableCount: number +): { + title: string; + description: string; +} { + return { + title: `Queue ${planId} inventory for removal?`, + description: + 'This will add the desired number of plans to the Pending Key Removal queue. After this, the admin must remove the key from the provider, and then mark the key as revoked', + }; +} + +export function canSubmitReduceCount(value: string, availableCount: number): boolean { + const count = Number(value); + return Number.isInteger(count) && count >= 1 && count <= availableCount; +} + +export function getReduceInventoryCompleteToast( + queuedCount: number, + requestedCount: number, + upstreamPlanIds: readonly string[] +): string { + if (queuedCount === 0) { + return 'No credentials were queued. They may have just been claimed by a subscription; refresh and try again.'; + } + const idList = upstreamPlanIds.join(', '); + if (queuedCount < requestedCount) { + return `Queued ${queuedCount} of ${requestedCount} requested credentials for removal (the rest were claimed before queuing). Find them in Pending Key Rotation and deprovision using upstream plan IDs: ${idList}`; + } + return `Queued ${queuedCount} credential${queuedCount === 1 ? '' : 's'} for removal in Pending Key Rotation. Deprovision using upstream plan IDs: ${idList}`; +} + export type InsightsRangeDays = 7 | 14 | 30; export type AdminSubscriptionSummary = { diff --git a/apps/web/src/lib/coding-plans/index.ts b/apps/web/src/lib/coding-plans/index.ts index 4c8fc77d91..bb065a08d2 100644 --- a/apps/web/src/lib/coding-plans/index.ts +++ b/apps/web/src/lib/coding-plans/index.ts @@ -44,10 +44,28 @@ export class CodingPlanInventoryReplacementError extends Error { } } +export class CodingPlanInventoryReductionError extends Error { + constructor(message: string) { + super(message); + this.name = 'CodingPlanInventoryReductionError'; + } +} + function inventoryReplacementError(message: string): CodingPlanInventoryReplacementError { return new CodingPlanInventoryReplacementError(message); } +export class CodingPlanCredentialReassignmentError extends Error { + constructor(message: string) { + super(message); + this.name = 'CodingPlanCredentialReassignmentError'; + } +} + +function credentialReassignmentError(message: string): CodingPlanCredentialReassignmentError { + return new CodingPlanCredentialReassignmentError(message); +} + function databaseConstraint(error: unknown): string | null { let current: unknown = error; for (let depth = 0; depth < 5 && current && typeof current === 'object'; depth++) { @@ -377,25 +395,24 @@ export async function terminateCodingPlanImmediately( subscriptionId: string, reason: CancellationReason = 'administrative_termination' ): Promise { - const [subscription] = await db - .select({ - id: coding_plan_subscriptions.id, - installed_byok_key_id: coding_plan_subscriptions.installed_byok_key_id, - key_inventory_id: coding_plan_subscriptions.key_inventory_id, - }) - .from(coding_plan_subscriptions) - .where( - and( - eq(coding_plan_subscriptions.id, subscriptionId), - inArray(coding_plan_subscriptions.status, ['active', 'past_due']) - ) - ) - .limit(1); - if (!subscription) { - throw new Error('No live subscription found.'); - } - await db.transaction(async tx => { + // Lock the subscription row before reading it so this can't act on a + // stale installed_byok_key_id / key_inventory_id snapshot if it races + // with a concurrent credential reassignment for the same subscription. + const [subscription] = await tx + .select({ + id: coding_plan_subscriptions.id, + status: coding_plan_subscriptions.status, + installed_byok_key_id: coding_plan_subscriptions.installed_byok_key_id, + key_inventory_id: coding_plan_subscriptions.key_inventory_id, + }) + .from(coding_plan_subscriptions) + .where(eq(coding_plan_subscriptions.id, subscriptionId)) + .for('update'); + if (!subscription || (subscription.status !== 'active' && subscription.status !== 'past_due')) { + throw new Error('No live subscription found.'); + } + await tx .update(coding_plan_subscriptions) .set({ @@ -438,6 +455,141 @@ export async function terminateCodingPlanImmediately( }); } +// Moves a live subscription off its currently assigned credential onto the +// next available one from the pool for the same plan/provider, then queues +// the vacated credential for manual upstream revocation. Unlike +// replaceInventoryCredential, this never requires an admin to supply key +// material: the pooled credential is decrypted server-side and re-encrypted +// straight into the existing installed BYOK row, so plaintext never leaves +// the transaction. Nothing here touches billing fields (cost, period dates, +// renewal, status) on coding_plan_subscriptions. +export async function reassignSubscriptionCredential( + subscriptionId: string, + adminUserId: string +): Promise<{ inventoryKeyId: string }> { + if (!BYOK_ENCRYPTION_KEY) { + throw credentialReassignmentError('BYOK encryption is not configured'); + } + + try { + const { newInventoryId, previousInventoryId } = await db.transaction(async tx => { + // Lock the subscription row before reading it so a concurrent + // cancellation/termination can't act on a stale key_inventory_id / + // installed_byok_key_id snapshot while this reassignment is in + // flight (and vice versa) - whichever transaction locks the row + // first always sees, and leaves, consistent data for the other. + const [subscription] = await tx + .select({ + id: coding_plan_subscriptions.id, + userId: coding_plan_subscriptions.user_id, + status: coding_plan_subscriptions.status, + planId: coding_plan_subscriptions.plan_id, + providerId: coding_plan_subscriptions.provider_id, + keyInventoryId: coding_plan_subscriptions.key_inventory_id, + installedByokKeyId: coding_plan_subscriptions.installed_byok_key_id, + }) + .from(coding_plan_subscriptions) + .where(eq(coding_plan_subscriptions.id, subscriptionId)) + .for('update'); + if ( + !subscription || + (subscription.status !== 'active' && subscription.status !== 'past_due') + ) { + throw new Error('No live subscription found.'); + } + const previousInventoryId = subscription.keyInventoryId; + const installedByokKeyId = subscription.installedByokKeyId; + if (!previousInventoryId || !installedByokKeyId) { + throw credentialReassignmentError('Subscription has no assigned credential to reassign.'); + } + + const { rows: lockedRows } = await tx.execute<{ id: string; status: string }>(sql` + SELECT id, status FROM coding_plan_key_inventory + WHERE id = ${previousInventoryId} + FOR UPDATE + `); + const locked = lockedRows[0]; + if (!locked || locked.status !== 'assigned') { + throw credentialReassignmentError('Credential is not eligible for reassignment.'); + } + + const { rows: claimedRows } = await tx.execute<{ + id: string; + encrypted_api_key: { iv: string; data: string; authTag: string } | null; + }>(sql` + UPDATE coding_plan_key_inventory + SET status = 'assigned', + assigned_to_user_id = ${subscription.userId}, + assigned_at = now(), + updated_at = now() + WHERE id = ( + SELECT id FROM coding_plan_key_inventory + WHERE plan_id = ${subscription.planId} + AND provider_id = ${subscription.providerId} + AND status = 'available' + LIMIT 1 + FOR UPDATE SKIP LOCKED + ) + RETURNING id, encrypted_api_key + `); + const claimed = claimedRows[0]; + if (!claimed?.encrypted_api_key) { + throw credentialReassignmentError( + `No available credential in the pool for plan "${subscription.planId}".` + ); + } + + const plaintext = decryptApiKey(claimed.encrypted_api_key, BYOK_ENCRYPTION_KEY); + const byokUpdateResult = await tx + .update(byok_api_keys) + .set({ encrypted_api_key: encryptApiKey(plaintext, BYOK_ENCRYPTION_KEY) }) + .where( + and( + eq(byok_api_keys.id, installedByokKeyId), + eq(byok_api_keys.management_source, 'coding_plan') + ) + ); + if ((byokUpdateResult.rowCount ?? 0) === 0) { + throw credentialReassignmentError( + 'Subscription credential could not be updated; it may have just been removed.' + ); + } + + await tx + .update(coding_plan_subscriptions) + .set({ key_inventory_id: claimed.id }) + .where(eq(coding_plan_subscriptions.id, subscription.id)); + + await tx + .update(coding_plan_key_inventory) + .set({ + status: 'revocation_pending', + encrypted_api_key: null, + revocation_requested_at: sql`now()`, + last_revocation_error: null, + }) + .where(eq(coding_plan_key_inventory.id, previousInventoryId)); + + return { newInventoryId: claimed.id, previousInventoryId }; + }); + + logInfo('Coding plan subscription credential reassigned from pool', { + subscriptionId, + previousInventoryId, + newInventoryId, + adminUserId, + }); + + return { inventoryKeyId: newInventoryId }; + } catch (error) { + if (error instanceof CodingPlanCredentialReassignmentError) throw error; + if (error instanceof Error && error.message === 'No live subscription found.') throw error; + throw credentialReassignmentError( + 'Unable to reassign the subscription credential due to a database error.' + ); + } +} + type InventoryCredentialValidator = ( input: CodingPlanCredentialValidationInput ) => Promise; @@ -656,6 +808,72 @@ export async function getKeyInventoryCounts( })); } +export async function queueAvailableInventoryForRevocation( + planId: CodingPlanId, + count: number, + adminUserId: string +): Promise<{ queued: Array<{ id: string; upstreamPlanId: string }> }> { + const plan = getCodingPlanPrice(planId); + if (!plan) { + throw new CodingPlanInventoryReductionError( + `Plan "${planId}" is not available as a coding plan.` + ); + } + + // Claims up to `count` unassigned, never-issued rows the same way + // subscribeToCodingPlan claims one: a raw UPDATE against a + // WHERE id IN (SELECT ... FOR UPDATE SKIP LOCKED) subquery, generalized + // from LIMIT 1 to LIMIT count; SKIP LOCKED lets this run safely alongside + // a concurrent subscribe claiming a row for the same plan. Claimed rows + // move into the same revocation_pending state that + // terminateCodingPlanImmediately and the billing-lifecycle-cron sweeps + // use, so they surface in the existing Pending Key Rotation queue with + // their retained upstream_plan_id for a human to deprovision in the + // provider's own console and confirm with Revoke/Replace; no separate + // admin queue is needed for this. Raw SQL bypasses Drizzle's + // $onUpdateFn, so updated_at is set explicitly here, matching + // subscribeToCodingPlan's claim query. assigned_to_user_id and + // assigned_at exclude anything already assigned, and the NOT EXISTS check + // only counts *live* (non-canceled) subscriptions, since a row recycled + // back to "available" via replaceManualCredentialRevocation can still + // carry a stale key_inventory_id pointer from its old, now-canceled + // subscription. ORDER BY created_at ASC claims the oldest stock first. + const { rows } = await db.execute<{ id: string; upstream_plan_id: string }>(sql` + UPDATE coding_plan_key_inventory + SET status = 'revocation_pending', + encrypted_api_key = NULL, + revocation_requested_at = now(), + updated_at = now() + WHERE id IN ( + SELECT id + FROM coding_plan_key_inventory + WHERE plan_id = ${plan.planId} + AND status = 'available' + AND assigned_to_user_id IS NULL + AND assigned_at IS NULL + AND NOT EXISTS ( + SELECT 1 + FROM coding_plan_subscriptions + WHERE coding_plan_subscriptions.key_inventory_id = coding_plan_key_inventory.id + AND coding_plan_subscriptions.status != 'canceled' + ) + ORDER BY created_at ASC + LIMIT ${count} + FOR UPDATE SKIP LOCKED + ) + RETURNING id, upstream_plan_id + `); + + const queued = rows.map(row => ({ id: row.id, upstreamPlanId: row.upstream_plan_id })); + logInfo('Coding plan available inventory queued for manual revocation', { + planId: plan.planId, + requested: count, + queued: queued.length, + adminUserId, + }); + return { queued }; +} + export async function adminCancelCodingPlanSubscription(subscriptionId: string): Promise { const result = await db .update(coding_plan_subscriptions) diff --git a/apps/web/src/routers/coding-plans-router.test.ts b/apps/web/src/routers/coding-plans-router.test.ts index 85b790b721..bc41b23af1 100644 --- a/apps/web/src/routers/coding-plans-router.test.ts +++ b/apps/web/src/routers/coding-plans-router.test.ts @@ -838,6 +838,107 @@ describe('coding plans router', () => { }); }); + it('queues available inventory for manual revocation by the requested count and returns their upstream plan IDs', async () => { + const admin = await insertTestUser({ is_admin: true }); + const caller = await createCallerForUser(admin.id); + const items = await Promise.all([insertInventory(), insertInventory(), insertInventory()]); + + const result = await caller.codingPlans.adminReduceInventory({ planId: PLAN_ID, count: 2 }); + + expect(result.queued).toHaveLength(2); + const queuedIds = result.queued.map(item => item.id); + expect(new Set(queuedIds).size).toBe(2); + expect(items.map(item => item.upstream_plan_id)).toEqual( + expect.arrayContaining(result.queued.map(item => item.upstreamPlanId)) + ); + + const rows = await db.select().from(coding_plan_key_inventory); + expect(rows).toHaveLength(3); + for (const row of rows) { + const wasQueued = queuedIds.includes(row.id); + expect(row.status).toBe(wasQueued ? 'revocation_pending' : 'available'); + expect(row.encrypted_api_key).toEqual(wasQueued ? null : expect.anything()); + if (wasQueued) { + expect(row.revocation_requested_at).not.toBeNull(); + expect(row.upstream_plan_id).toBe( + result.queued.find(item => item.id === row.id)?.upstreamPlanId + ); + } + } + + const queue = await caller.codingPlans.adminRevocationQueue({}); + expect(queue.map(item => item.inventoryKeyId).sort()).toEqual(queuedIds.sort()); + }); + + it('queues fewer than requested when not enough available inventory exists for the plan', async () => { + const admin = await insertTestUser({ is_admin: true }); + const caller = await createCallerForUser(admin.id); + await insertInventory(); + + const result = await caller.codingPlans.adminReduceInventory({ planId: PLAN_ID, count: 5 }); + + expect(result.queued).toHaveLength(1); + const [row] = await db.select().from(coding_plan_key_inventory); + expect(row.status).toBe('revocation_pending'); + }); + + it('never queues assigned inventory or inventory referenced by a live subscription', async () => { + const admin = await insertTestUser({ is_admin: true }); + const user = await insertTestUser(); + const caller = await createCallerForUser(admin.id); + const assigned = await insertInventory({ + status: 'assigned', + assigned_to_user_id: user.id, + assigned_at: new Date().toISOString(), + }); + const pendingRevocation = await insertInventory({ + status: 'revocation_pending', + revocation_requested_at: new Date().toISOString(), + }); + await db + .insert(coding_plan_subscriptions) + .values(subscriptionValues(user.id, { key_inventory_id: assigned.id, status: 'active' })); + + const result = await caller.codingPlans.adminReduceInventory({ planId: PLAN_ID, count: 5 }); + + expect(result.queued).toHaveLength(0); + const rowsById = new Map( + (await db.select().from(coding_plan_key_inventory)).map(row => [row.id, row]) + ); + expect(rowsById.get(assigned.id)?.status).toBe('assigned'); + expect(rowsById.get(pendingRevocation.id)?.status).toBe('revocation_pending'); + }); + + it('queues an available credential even when a canceled subscription still references it via a stale key_inventory_id', async () => { + const admin = await insertTestUser({ is_admin: true }); + const user = await insertTestUser(); + const caller = await createCallerForUser(admin.id); + // Simulates a credential that was assigned, revoked, and manually + // recycled back to "available" via replaceManualCredentialRevocation. + // The original (now canceled) subscription's key_inventory_id is never + // cleared, so this row is still "available" but has a stale FK pointing + // at it from a subscription that is no longer live. + const recycled = await insertInventory({ status: 'available' }); + await db + .insert(coding_plan_subscriptions) + .values(subscriptionValues(user.id, { key_inventory_id: recycled.id, status: 'canceled' })); + + const result = await caller.codingPlans.adminReduceInventory({ planId: PLAN_ID, count: 1 }); + + expect(result.queued).toHaveLength(1); + expect(result.queued[0].id).toBe(recycled.id); + }); + + it('rejects non-admin callers reducing inventory', async () => { + const user = await insertTestUser(); + const caller = await createCallerForUser(user.id); + await insertInventory(); + + await expect( + caller.codingPlans.adminReduceInventory({ planId: PLAN_ID, count: 1 }) + ).rejects.toThrow(); + }); + it('restricts manual remediation and returns the upstream identifier needed to deprovision', async () => { const admin = await insertTestUser({ is_admin: true }); const user = await insertTestUser(); diff --git a/apps/web/src/routers/coding-plans-router.ts b/apps/web/src/routers/coding-plans-router.ts index c42b45897f..9282092b22 100644 --- a/apps/web/src/routers/coding-plans-router.ts +++ b/apps/web/src/routers/coding-plans-router.ts @@ -5,6 +5,8 @@ import * as z from 'zod'; import { adminCancelCodingPlanSubscription, cancelCodingPlanSubscription, + CodingPlanCredentialReassignmentError, + CodingPlanInventoryReductionError, CodingPlanInventoryReplacementError, CodingPlanInventoryUploadError, extendCodingPlanSubscriptionPeriod, @@ -12,6 +14,8 @@ import { getCodingPlanAvailabilityIntentCounts, getCodingPlanAvailabilityIntentPlanIds, getKeyInventoryCounts, + reassignSubscriptionCredential, + queueAvailableInventoryForRevocation, replaceInventoryCredential, requestCodingPlanAvailabilityNotification, subscribeToCodingPlan, @@ -811,6 +815,25 @@ export const codingPlansRouter = createTRPCRouter({ } }), + adminReduceInventory: adminProcedure + .input( + z.object({ + planId: CodingPlanIdSchema, + count: z.number().int().min(1).max(1000), + }) + ) + .mutation(async ({ input, ctx }) => { + try { + return await queueAvailableInventoryForRevocation(input.planId, input.count, ctx.user.id); + } catch (error) { + const message = + error instanceof CodingPlanInventoryReductionError + ? error.message + : 'Unable to reduce Coding Plan inventory.'; + throw new TRPCError({ code: 'INTERNAL_SERVER_ERROR', message }); + } + }), + adminTerminateSubscription: adminProcedure .input(z.object({ subscriptionId: SubscriptionIdSchema })) .mutation(async ({ input }) => { @@ -880,6 +903,26 @@ export const codingPlansRouter = createTRPCRouter({ } }), + adminReassignSubscriptionCredential: adminProcedure + .input(z.object({ subscriptionId: SubscriptionIdSchema })) + .mutation(async ({ input, ctx }) => { + try { + return await reassignSubscriptionCredential(input.subscriptionId, ctx.user.id); + } catch (error) { + if (error instanceof CodingPlanCredentialReassignmentError) { + throw new TRPCError({ code: 'PRECONDITION_FAILED', message: error.message }); + } + const message = error instanceof Error ? error.message : String(error); + if (message.includes('No live subscription')) { + throw new TRPCError({ code: 'NOT_FOUND', message }); + } + throw new TRPCError({ + code: 'INTERNAL_SERVER_ERROR', + message: 'Unable to reassign the subscription credential.', + }); + } + }), + adminRevocationQueue: adminProcedure .input( z.object({ From 2e27080077a2b03fcdb0dfb7f67f47c91c7e80fc Mon Sep 17 00:00:00 2001 From: Anil Kulkarni Date: Fri, 18 Sep 2026 10:34:21 -0700 Subject: [PATCH 3/3] linting --- .../coding-plans/CodingPlansOperationsContent.tsx | 5 +---- .../coding-plans/coding-plan-operations.test.ts | 14 ++++---------- .../admin/coding-plans/coding-plan-operations.ts | 5 +---- 3 files changed, 6 insertions(+), 18 deletions(-) diff --git a/apps/web/src/app/admin/coding-plans/CodingPlansOperationsContent.tsx b/apps/web/src/app/admin/coding-plans/CodingPlansOperationsContent.tsx index a3553dc6ab..14e99e58db 100644 --- a/apps/web/src/app/admin/coding-plans/CodingPlansOperationsContent.tsx +++ b/apps/web/src/app/admin/coding-plans/CodingPlansOperationsContent.tsx @@ -1845,10 +1845,7 @@ function OperationsDialogs({ const swapCopy = getSwapCredentialDialogCopy(swapSelection?.userName ?? 'this user'); const parsedExtendDays = Number(extendDays); const inventoryReplacementCopy = getInventoryReplacementDialogCopy(inventoryReplacementId); - const reduceCopy = getReduceInventoryDialogCopy( - reduceSelection?.planId ?? 'this plan', - reduceSelection?.availableCount ?? 0 - ); + const reduceCopy = getReduceInventoryDialogCopy(reduceSelection?.planId ?? 'this plan'); const parsedReduceCount = Number(reduceCount); return ( diff --git a/apps/web/src/app/admin/coding-plans/coding-plan-operations.test.ts b/apps/web/src/app/admin/coding-plans/coding-plan-operations.test.ts index 74067130bd..fb4e979f1f 100644 --- a/apps/web/src/app/admin/coding-plans/coding-plan-operations.test.ts +++ b/apps/web/src/app/admin/coding-plans/coding-plan-operations.test.ts @@ -121,17 +121,11 @@ describe('subscription action copy', () => { }); describe('reduce inventory copy', () => { - it('names the plan and available count, and explains the manual provider-side step', () => { - const dialog = getReduceInventoryDialogCopy('minimax-token-plan-plus', 5); + it('names the plan and explains the manual provider-side step', () => { + const dialog = getReduceInventoryDialogCopy('minimax-token-plan-plus'); expect(dialog.title).toBe('Queue minimax-token-plan-plus inventory for removal?'); - expect(dialog.description).toContain('5 credentials are currently available'); - expect(dialog.description).toContain('Pending Key Rotation'); - expect(dialog.description).toContain('Kilo never contacts the provider'); - }); - - it('uses singular phrasing for a single available credential', () => { - const dialog = getReduceInventoryDialogCopy('minimax-token-plan-plus', 1); - expect(dialog.description).toContain('1 credential is currently available'); + expect(dialog.description).toContain('Pending Key Removal queue'); + expect(dialog.description).toContain('remove the key from the provider'); }); it('bounds the requested count to a positive integer no greater than what is available', () => { diff --git a/apps/web/src/app/admin/coding-plans/coding-plan-operations.ts b/apps/web/src/app/admin/coding-plans/coding-plan-operations.ts index 6327f4cfcb..32d46c367a 100644 --- a/apps/web/src/app/admin/coding-plans/coding-plan-operations.ts +++ b/apps/web/src/app/admin/coding-plans/coding-plan-operations.ts @@ -107,10 +107,7 @@ export function getSwapCredentialCompleteToast(): string { return 'Subscription moved to a new pooled credential.'; } -export function getReduceInventoryDialogCopy( - planId: string, - availableCount: number -): { +export function getReduceInventoryDialogCopy(planId: string): { title: string; description: string; } {