diff --git a/cloudflare/packages/worker/src/index.ts b/cloudflare/packages/worker/src/index.ts index 9c6e690a9..6ca348eed 100644 --- a/cloudflare/packages/worker/src/index.ts +++ b/cloudflare/packages/worker/src/index.ts @@ -46,6 +46,12 @@ import { scheduleAxiomSpanExport, type MutableRequestTelemetryContext, } from "./telemetry.ts" +import { + migrateLegacyTenant, + rollbackLegacyMigrationDestination, + validatedLegacyMigrationBinding, + type LegacyMigrationAccount, +} from "./legacy-migration.ts" export { TenantRegistryDurableObject } @@ -273,6 +279,24 @@ interface AccountRow { readonly consecutive_quota_errors: number | null } +interface HostedMigrationStateRow { + readonly [key: string]: SqlStorageValue + readonly org_id: string + readonly accounts_json: string + readonly phase: string + readonly destination_origin: string + readonly tenant_key_sha256: string + readonly migration_id: string + readonly account_count: number + readonly started_at: number +} + +interface HostedMigrationBinding { + readonly destinationOrigin: string + readonly tenantKeySha256: string + readonly migrationId: string +} + interface TableInfoRow { readonly [key: string]: SqlStorageValue readonly name: string @@ -436,6 +460,48 @@ const parseJsonRecord = async ( return body as Record } +const parseBoundedJsonRecord = async ( + request: Request, + maxBytes: number +): Promise | Response> => { + const contentLength = Number(request.headers.get("content-length") ?? "0") + if (Number.isFinite(contentLength) && contentLength > maxBytes) { + return json({ error: "Request body too large" }, { status: 413 }) + } + if (!request.body) { + return json({ error: "Missing JSON body" }, { status: 400 }) + } + const reader = request.body.getReader() + const chunks: Uint8Array[] = [] + let total = 0 + for (;;) { + const { done, value } = await reader.read() + if (done) break + total += value.byteLength + if (total > maxBytes) { + await reader.cancel().catch(() => {}) + return json({ error: "Request body too large" }, { status: 413 }) + } + chunks.push(value) + } + const body = new Uint8Array(total) + let offset = 0 + for (const chunk of chunks) { + body.set(chunk, offset) + offset += chunk.byteLength + } + let decoded: unknown + try { + decoded = JSON.parse(new TextDecoder().decode(body)) + } catch { + return json({ error: "Missing JSON body" }, { status: 400 }) + } + if (!decoded || typeof decoded !== "object" || Array.isArray(decoded)) { + return json({ error: "Missing JSON body" }, { status: 400 }) + } + return decoded as Record +} + const parseRouteInput = async (request: Request): Promise => { const body = await request.json().catch(() => ({})) if (!body || typeof body !== "object" || Array.isArray(body)) { @@ -1635,6 +1701,7 @@ const generateTotp = async ( export class SubrouterDurableObject extends DurableObject { private readonly refreshInFlight = new Map>() + private migrationPreparing = false constructor(ctx: DurableObjectState, env: Env) { super(ctx, env) @@ -1753,12 +1820,23 @@ export class SubrouterDurableObject extends DurableObject { draining INTEGER NOT NULL DEFAULT 0, started_at INTEGER NOT NULL ); + CREATE TABLE IF NOT EXISTS hosted_migration_state( + org_id TEXT PRIMARY KEY, + accounts_json TEXT NOT NULL, + phase TEXT NOT NULL, + destination_origin TEXT NOT NULL, + tenant_key_sha256 TEXT NOT NULL, + migration_id TEXT NOT NULL, + account_count INTEGER NOT NULL, + started_at INTEGER NOT NULL + ); INSERT OR IGNORE INTO subrouter_status(id) VALUES (1); INSERT OR IGNORE INTO lifecycle(id, started_at) VALUES (1, ${Date.now()}); `) this.ensureAccountHealthColumns() this.ctx.blockConcurrencyWhile(async () => { + this.restoreInterruptedHostedMigrations() this.pruneCredentialLeases(Date.now()) await this.scheduleNextRefreshAlarm() }) @@ -2350,6 +2428,7 @@ export class SubrouterDurableObject extends DurableObject { } async upsertAccount(input: UpsertAccountInput): Promise<{ account: StoredAccount }> { + this.assertLegacySourceWritable(input.orgId) const sql = this.ctx.storage.sql const now = Date.now() sql.exec( @@ -2406,6 +2485,7 @@ export class SubrouterDurableObject extends DurableObject { readonly accountId: string }): Promise<{ account: StoredAccount }> { const orgId = this.resolveOrgId(input.orgId) + this.assertLegacySourceWritable(orgId) const accountId = this.requireNonEmpty(input.accountId, "accountId") const row = this.getAccountRow( this.ctx.storage.sql, @@ -2479,40 +2559,254 @@ export class SubrouterDurableObject extends DurableObject { } } + async migrateHosted(input: { + readonly orgId: string + readonly destinationUrl: unknown + readonly tenantKey: unknown + readonly finalizeSource: boolean + readonly allowLoopback: boolean + }): Promise<{ migrated: number }> { + const orgId = this.resolveOrgId(input.orgId) + const migrationId = `legacy-${orgId}` + const validated = validatedLegacyMigrationBinding({ + destinationUrl: input.destinationUrl, + tenantKey: input.tenantKey, + migrationId, + allowLoopback: input.allowLoopback, + }) + const binding: HostedMigrationBinding = { + destinationOrigin: validated.destinationUrl, + tenantKeySha256: await tenantKeySha256Hex(validated.tenantKey), + migrationId: validated.migrationId, + } + const migrate = async (): Promise<{ migrated: number }> => ({ + migrated: await migrateLegacyTenant({ + destinationUrl: validated.destinationUrl, + tenantKey: validated.tenantKey, + finalizeSource: input.finalizeSource, + allowLoopback: input.allowLoopback, + migrationId: validated.migrationId, + source: { + list: async () => this.listMigrationAccounts(orgId), + begin: async () => this.beginMigrationAccounts(orgId, binding), + complete: async (accounts) => + this.completeMigrationAccounts(orgId, accounts), + restore: async (_accounts, options) => + this.restoreMigrationAccounts(orgId, options.preserveRecovery), + markActivating: async () => + this.markMigrationDestinationActivating(orgId), + }, + }), + }) + if (this.migrationPreparing) { + throw new Error("hosted migration is already in progress") + } + this.migrationPreparing = true + try { + if (!input.finalizeSource) return await migrate() + const receipt = this.hostedMigrationState(orgId) + if (receipt?.phase === "completed") { + this.assertHostedMigrationBinding(receipt, binding) + const sourceAccounts = this.ctx.storage.sql + .exec( + "SELECT COUNT(*) AS count FROM accounts WHERE org_id = ?", + orgId + ) + .one().count + if (sourceAccounts !== 0) { + throw new Error("legacy source changed after hosted migration completed") + } + return { migrated: receipt.account_count } + } + + // Stop new refreshes before waiting for the ones that already own a + // single-use refresh token. No source row changes until they settle. + const pending = this.hostedMigrationState(orgId) + if (pending) { + this.assertHostedMigrationBinding(pending, binding) + } + if (pending) { + await rollbackLegacyMigrationDestination({ + destinationUrl: validated.destinationUrl, + tenantKey: validated.tenantKey, + migrationId: validated.migrationId, + allowLoopback: input.allowLoopback, + }) + this.restoreMigrationAccounts(orgId) + } + await Promise.allSettled([...this.refreshInFlight.values()]) + // Cloudflare bounds this barrier at 30 seconds. Migration uploads run in + // parallel with a 10-second request deadline, leaving room for the two + // local transactions while excluding every concurrent tenant mutation. + return await this.ctx.blockConcurrencyWhile(migrate) + } finally { + this.migrationPreparing = false + } + } + + private listMigrationAccounts( + orgId: string + ): ReadonlyArray { + return this.ctx.storage.sql + .exec( + `SELECT * FROM accounts + WHERE org_id = ? + ORDER BY id`, + orgId + ) + .toArray() + .map((row) => ({ + id: row.id, + kind: row.kind as AccountKind, + label: row.label, + enabled: row.enabled === 1, + hasTotp: row.totp_seed_base32 !== null, + updatedAt: row.updated_at, + ...(row.credentials_json + ? { credentials: JSON.parse(row.credentials_json) as AccountCredentials } + : {}), + })) + } + + private beginMigrationAccounts( + orgId: string, + binding: HostedMigrationBinding + ): ReadonlyArray { + const accounts = this.listMigrationAccounts(orgId) + const accountsJSON = JSON.stringify(accounts) + this.ctx.storage.transactionSync(() => { + const existing = this.ctx.storage.sql + .exec( + "SELECT COUNT(*) AS count FROM hosted_migration_state WHERE org_id = ?", + orgId + ) + .one().count + if (existing !== 0) { + throw new Error("hosted migration recovery is pending") + } + this.ctx.storage.sql.exec( + `INSERT INTO hosted_migration_state( + org_id, + accounts_json, + phase, + destination_origin, + tenant_key_sha256, + migration_id, + account_count, + started_at + ) VALUES (?, ?, 'quiesced', ?, ?, ?, ?, ?)`, + orgId, + accountsJSON, + binding.destinationOrigin, + binding.tenantKeySha256, + binding.migrationId, + accounts.length, + Date.now() + ) + this.ctx.storage.sql.exec( + "UPDATE accounts SET enabled = 0 WHERE org_id = ?", + orgId + ) + }) + return accounts + } + + private restoreMigrationAccounts( + orgId: string, + preserveRecovery = false + ): void { + this.ctx.storage.transactionSync(() => { + const state = this.hostedMigrationState(orgId) + if (!state) return + const accounts = JSON.parse( + state.accounts_json + ) as ReadonlyArray + for (const account of accounts) { + this.ctx.storage.sql.exec( + "UPDATE accounts SET enabled = ? WHERE id = ? AND org_id = ?", + account.enabled ? 1 : 0, + account.id, + orgId + ) + } + if (preserveRecovery) { + this.ctx.storage.sql.exec( + `UPDATE hosted_migration_state + SET phase = 'cleanup_pending' + WHERE org_id = ?`, + orgId + ) + } else { + this.ctx.storage.sql.exec( + "DELETE FROM hosted_migration_state WHERE org_id = ?", + orgId + ) + } + }) + } + + private markMigrationDestinationActivating(orgId: string): void { + this.ctx.storage.transactionSync(() => { + const state = this.hostedMigrationState(orgId) + if (!state || state.phase !== "quiesced") { + throw new Error("hosted migration source is not quiesced") + } + this.ctx.storage.sql.exec( + `UPDATE hosted_migration_state + SET phase = 'activating' + WHERE org_id = ?`, + orgId + ) + }) + } + + private completeMigrationAccounts( + orgId: string, + accounts: ReadonlyArray + ): void { + this.ctx.storage.transactionSync(() => { + const sql = this.ctx.storage.sql + const state = this.hostedMigrationState(orgId) + if ( + !state || + state.phase !== "activating" || + state.accounts_json !== JSON.stringify(accounts) + ) { + throw new Error("hosted migration source snapshot is unavailable") + } + const rows = sql + .exec( + "SELECT * FROM accounts WHERE org_id = ? ORDER BY id", + orgId + ) + .toArray() + if ( + rows.length !== accounts.length || + rows.some((row, index) => + !this.migrationAccountMatchesRow(accounts[index], row) + ) + ) { + throw new Error("migration source changed during hosted upload") + } + for (const account of accounts) { + this.deleteAccountRows(sql, orgId, account.id) + } + sql.exec( + `UPDATE hosted_migration_state + SET accounts_json = '[]', phase = 'completed' + WHERE org_id = ?`, + orgId + ) + }) + } + async deleteAccount(input: { readonly orgId?: string; readonly accountId: string }): Promise<{ ok: true }> { const orgId = this.resolveOrgId(input.orgId) + this.assertLegacySourceWritable(orgId) const accountId = this.requireNonEmpty(input.accountId, "accountId") const row = this.getAccountRow(this.ctx.storage.sql, orgId, accountId, false) if (!row) throw new Error("account not found") - const sql = this.ctx.storage.sql - sql.exec("DELETE FROM accounts WHERE id = ? AND org_id = ?", accountId, orgId) - sql.exec("DELETE FROM account_model_quotas WHERE account_id = ?", accountId) - sql.exec("DELETE FROM account_usage WHERE account_id = ?", accountId) - sql.exec( - "DELETE FROM session_assignments WHERE org_id = ? AND account_id = ?", - orgId, - accountId - ) - sql.exec( - "DELETE FROM sticky_sessions WHERE org_id = ? AND account_id = ?", - orgId, - accountId - ) - sql.exec( - "DELETE FROM sticky_session_routes WHERE org_id = ? AND account_id = ?", - orgId, - accountId - ) - sql.exec( - "DELETE FROM credential_leases WHERE org_id = ? AND account_id = ?", - orgId, - accountId - ) - sql.exec( - "DELETE FROM credential_quota_cooldowns WHERE org_id = ? AND account_id = ?", - orgId, - accountId - ) + this.deleteAccountRows(this.ctx.storage.sql, orgId, accountId) await this.scheduleNextRefreshAlarm() return { ok: true } } @@ -2523,6 +2817,7 @@ export class SubrouterDurableObject extends DurableObject { readonly replacement: UpsertAccountInput }): Promise<{ account: StoredAccount }> { const orgId = this.resolveOrgId(input.orgId) + this.assertLegacySourceWritable(orgId) const accountId = this.requireNonEmpty(input.accountId, "accountId") const existing = this.getAccountRow( this.ctx.storage.sql, @@ -3133,6 +3428,123 @@ export class SubrouterDurableObject extends DurableObject { ) } + private hostedMigrationState( + orgId: string + ): HostedMigrationStateRow | null { + return this.ctx.storage.sql + .exec( + "SELECT * FROM hosted_migration_state WHERE org_id = ?", + orgId + ) + .toArray()[0] ?? null + } + + private assertLegacySourceWritable(orgId: string): void { + if (this.hostedMigrationState(orgId)) { + throw new Error( + "legacy source is read-only during or after hosted migration" + ) + } + } + + private assertHostedMigrationBinding( + state: HostedMigrationStateRow, + binding: HostedMigrationBinding + ): void { + if ( + state.destination_origin !== binding.destinationOrigin || + state.tenant_key_sha256 !== binding.tenantKeySha256 || + state.migration_id !== binding.migrationId + ) { + throw new Error("hosted migration recovery destination does not match") + } + } + + private migrationAccountMatchesRow( + account: LegacyMigrationAccount | undefined, + row: AccountRow + ): boolean { + if (!account) return false + const credentialsJSON = account.credentials === undefined + ? null + : JSON.stringify(account.credentials) + return ( + row.id === account.id && + row.kind === account.kind && + row.label === account.label && + row.enabled === 0 && + (row.totp_seed_base32 !== null) === account.hasTotp && + row.credentials_json === credentialsJSON && + row.updated_at === account.updatedAt + ) + } + + private restoreInterruptedHostedMigrations(): void { + const states = this.ctx.storage.sql + .exec( + "SELECT * FROM hosted_migration_state ORDER BY org_id" + ) + .toArray() + if (states.length === 0) return + this.ctx.storage.transactionSync(() => { + for (const state of states) { + if (state.phase !== "quiesced") continue + const accounts = JSON.parse( + state.accounts_json + ) as ReadonlyArray + for (const account of accounts) { + this.ctx.storage.sql.exec( + "UPDATE accounts SET enabled = ? WHERE id = ? AND org_id = ?", + account.enabled ? 1 : 0, + account.id, + state.org_id + ) + } + this.ctx.storage.sql.exec( + `UPDATE hosted_migration_state + SET phase = 'cleanup_pending' + WHERE org_id = ?`, + state.org_id + ) + } + }) + } + + private deleteAccountRows( + sql: SqlStorage, + orgId: string, + accountId: string + ): void { + sql.exec("DELETE FROM accounts WHERE id = ? AND org_id = ?", accountId, orgId) + sql.exec("DELETE FROM account_model_quotas WHERE account_id = ?", accountId) + sql.exec("DELETE FROM account_usage WHERE account_id = ?", accountId) + sql.exec( + "DELETE FROM session_assignments WHERE org_id = ? AND account_id = ?", + orgId, + accountId + ) + sql.exec( + "DELETE FROM sticky_sessions WHERE org_id = ? AND account_id = ?", + orgId, + accountId + ) + sql.exec( + "DELETE FROM sticky_session_routes WHERE org_id = ? AND account_id = ?", + orgId, + accountId + ) + sql.exec( + "DELETE FROM credential_leases WHERE org_id = ? AND account_id = ?", + orgId, + accountId + ) + sql.exec( + "DELETE FROM credential_quota_cooldowns WHERE org_id = ? AND account_id = ?", + orgId, + accountId + ) + } + private async refreshAccountIfExpired( orgId: string, accountId: string, @@ -3149,6 +3561,7 @@ export class SubrouterDurableObject extends DurableObject { const key = `${orgId}:${accountId}` const existing = this.refreshInFlight.get(key) if (existing) return existing + if (this.migrationPreparing) return row const promise = this.refreshAccountRow(row, force).finally(() => { this.refreshInFlight.delete(key) }) @@ -4856,6 +5269,42 @@ const handleFetch = async ( return json(await registryActor(env).listTenants()) } + const tenantMigrationMatch = url.pathname.match( + /^\/admin\/tenants\/([^/]+)\/migrate-hosted$/ + ) + if (tenantMigrationMatch && request.method === "POST") { + const tenantId = decodeURIComponent(tenantMigrationMatch[1]!) + const record = await parseBoundedJsonRecord(request, 4 * 1024) + if (record instanceof Response) return record + if ( + record["finalizeSource"] !== undefined && + typeof record["finalizeSource"] !== "boolean" + ) { + return json({ error: "finalizeSource must be boolean" }, { status: 400 }) + } + try { + const requestHost = new URL(request.url).hostname + const finalizeSource = record["finalizeSource"] === true + const { migrated } = await adminActor(env, tenantId).migrateHosted({ + orgId: tenantId, + destinationUrl: record["destinationUrl"], + tenantKey: record["tenantKey"], + finalizeSource, + allowLoopback: + requestHost === "localhost" || + requestHost === "127.0.0.1" || + requestHost === "[::1]", + }) + return json( + { ok: true, migrated, sourceFinalized: finalizeSource }, + { headers: { "Cache-Control": "no-store" } } + ) + } catch (error) { + if (telemetry) telemetry.errorType = errorTypeFor(error) + return errorJson(error, 409) + } + } + const tenantRevokeMatch = url.pathname.match(/^\/admin\/tenants\/([^/]+)\/revoke$/) if (tenantRevokeMatch && request.method === "POST") { const tenantId = decodeURIComponent(tenantRevokeMatch[1]!) diff --git a/cloudflare/packages/worker/src/legacy-migration.ts b/cloudflare/packages/worker/src/legacy-migration.ts new file mode 100644 index 000000000..a1e1f1793 --- /dev/null +++ b/cloudflare/packages/worker/src/legacy-migration.ts @@ -0,0 +1,423 @@ +import type { AccountCredentials, AccountKind } from "./contract.ts" + +export interface LegacyMigrationAccount { + readonly id: string + readonly kind: AccountKind + readonly label: string + readonly enabled: boolean + readonly hasTotp: boolean + readonly updatedAt?: number + readonly credentials?: AccountCredentials +} + +export interface LegacyMigrationSource { + readonly list: () => Promise> + readonly begin: () => Promise> + readonly complete: ( + accounts: ReadonlyArray + ) => Promise + readonly restore: ( + accounts: ReadonlyArray, + options: { readonly preserveRecovery: boolean } + ) => Promise + readonly markActivating?: () => Promise +} + +const productionOrigins = new Set([ + "https://sr.cmux.com", + "https://sr.cmux.dev", +]) +const tenantKeyPattern = /^srt_[0-9a-f]{32}$/ +const migrationIdPattern = /^[a-z0-9][a-z0-9._-]{0,159}$/ +const maxMigrationAccounts = 16 + +export function validatedLegacyMigrationBinding(options: { + readonly destinationUrl: unknown + readonly tenantKey: unknown + readonly migrationId?: string + readonly allowLoopback?: boolean +}): { + readonly destinationUrl: string + readonly tenantKey: string + readonly migrationId: string +} { + return { + destinationUrl: migrationDestination( + options.destinationUrl, + options.allowLoopback ?? false + ), + tenantKey: migrationTenantKey(options.tenantKey), + migrationId: normalizedMigrationID(options.migrationId), + } +} + +export async function migrateLegacyTenant(options: { + readonly destinationUrl: unknown + readonly tenantKey: unknown + readonly finalizeSource: boolean + readonly allowLoopback?: boolean + readonly migrationId?: string + readonly source: LegacyMigrationSource + readonly fetch?: typeof fetch +}): Promise { + // Reject operator input before quiescing any source credential. + const { destinationUrl, tenantKey, migrationId } = + validatedLegacyMigrationBinding(options) + let accounts: ReadonlyArray = [] + let sourceBegan = false + let destinationAttempted = false + let activationAttempted = false + let rollbackConfirmed = false + let sourceCompleted = false + try { + if (options.finalizeSource) { + validatedMigrationUploads(await options.source.list()) + // Mark the attempt before calling begin so a source with persisted + // recovery state can undo a failure that happens after quiescing. + sourceBegan = true + accounts = await options.source.begin() + } else { + accounts = await options.source.list() + } + const migrated = await migrateLegacyAccountsToHosted({ + destinationUrl, + tenantKey, + migrationId, + accounts, + allowLoopback: options.allowLoopback, + fetch: options.fetch, + onRequestStart: () => { + destinationAttempted = true + }, + }) + if (sourceBegan) { + await options.source.markActivating?.() + await activateLegacyMigrationDestination({ + destinationUrl, + tenantKey, + migrationId, + accountIds: accounts.map((account) => account.id), + allowLoopback: options.allowLoopback, + fetch: options.fetch, + onRequestStart: () => { + activationAttempted = true + }, + }) + await options.source.complete(accounts) + sourceCompleted = true + } + return migrated + } catch (error) { + if (destinationAttempted && !sourceCompleted) { + if (isAuthoritativeStageRejection(error)) { + rollbackConfirmed = true + } else { + try { + await rollbackLegacyMigrationDestination({ + destinationUrl, + tenantKey, + migrationId, + allowLoopback: options.allowLoopback, + fetch: options.fetch, + }) + rollbackConfirmed = true + } catch { + if (activationAttempted) { + throw new Error("hosted migration failed and destination rollback failed") + } + } + } + } + if (sourceBegan && !sourceCompleted) { + try { + await options.source.restore(accounts, { + preserveRecovery: destinationAttempted && !rollbackConfirmed, + }) + } catch { + throw new Error("hosted migration failed and source restoration failed") + } + } + throw error + } +} + +export async function migrateLegacyAccountsToHosted(options: { + readonly destinationUrl: unknown + readonly tenantKey: unknown + readonly migrationId?: string + readonly accounts: ReadonlyArray + readonly allowLoopback?: boolean + readonly fetch?: typeof fetch + readonly onRequestStart?: () => void +}): Promise { + const destination = migrationDestination( + options.destinationUrl, + options.allowLoopback ?? false + ) + const tenantKey = migrationTenantKey(options.tenantKey) + const migrationId = normalizedMigrationID(options.migrationId) + // Validate every source row before the first external mutation. A retry can + // safely overwrite the same destination ids after a network interruption. + const uploads = validatedMigrationUploads(options.accounts) + const accountIds = uploads.map((upload) => String(upload.accountId)) + const body = await migrationDestinationRequest({ + destination, + tenantKey, + path: "/_subrouter/accounts/migration/stage", + body: { migrationId, accounts: uploads }, + fetch: options.fetch, + operation: "stage", + onRequestStart: options.onRequestStart, + }) + if ( + body["ok"] !== true || + !Array.isArray(body["accountIds"]) || + body["accountIds"].length !== accountIds.length || + body["accountIds"].some( + (accountId, index) => accountId !== accountIds[index] + ) + ) { + throw new Error("hosted migration returned an invalid staged batch") + } + return uploads.length +} + +async function activateLegacyMigrationDestination(options: { + readonly destinationUrl: unknown + readonly tenantKey: unknown + readonly migrationId?: string + readonly accountIds: ReadonlyArray + readonly allowLoopback?: boolean + readonly fetch?: typeof fetch + readonly onRequestStart?: () => void +}): Promise { + const body = await migrationDestinationRequest({ + destination: migrationDestination( + options.destinationUrl, + options.allowLoopback ?? false + ), + tenantKey: migrationTenantKey(options.tenantKey), + path: "/_subrouter/accounts/migration/activate", + body: { + migrationId: normalizedMigrationID(options.migrationId), + accountIds: options.accountIds, + }, + fetch: options.fetch, + operation: "activation", + onRequestStart: options.onRequestStart, + }) + if (body["ok"] !== true) { + throw new Error("hosted migration returned an invalid activation") + } +} + +export async function rollbackLegacyMigrationDestination(options: { + readonly destinationUrl: unknown + readonly tenantKey: unknown + readonly migrationId?: string + readonly allowLoopback?: boolean + readonly fetch?: typeof fetch +}): Promise { + const body = await migrationDestinationRequest({ + destination: migrationDestination( + options.destinationUrl, + options.allowLoopback ?? false + ), + tenantKey: migrationTenantKey(options.tenantKey), + path: "/_subrouter/accounts/migration/rollback", + body: { migrationId: normalizedMigrationID(options.migrationId) }, + fetch: options.fetch, + operation: "rollback", + }) + if (body["ok"] !== true) { + throw new Error("hosted migration returned an invalid rollback") + } +} + +async function migrationDestinationRequest(options: { + readonly destination: string + readonly tenantKey: string + readonly path: string + readonly body: Record + readonly fetch?: typeof fetch + readonly operation: string + readonly onRequestStart?: () => void +}): Promise> { + let response: Response + try { + options.onRequestStart?.() + response = await (options.fetch ?? fetch)( + `${options.destination}${options.path}`, + { + method: "POST", + redirect: "manual", + signal: AbortSignal.timeout(10_000), + headers: { + authorization: `Bearer ${options.tenantKey}`, + "content-type": "application/json", + }, + body: JSON.stringify(options.body), + } + ) + } catch { + throw new Error("hosted migration destination is unavailable") + } + if (!response.ok) { + throw new MigrationDestinationResponseError( + options.operation, + response.status + ) + } + const body = await response.json().catch(() => null) + if (!body || typeof body !== "object" || Array.isArray(body)) { + throw new Error("hosted migration returned an invalid response") + } + return body as Record +} + +class MigrationDestinationResponseError extends Error { + constructor( + readonly operation: string, + readonly status: number + ) { + super(`hosted migration ${operation} failed (${status})`) + this.name = "MigrationDestinationResponseError" + } +} + +function isAuthoritativeStageRejection(error: unknown): boolean { + return ( + error instanceof MigrationDestinationResponseError && + error.operation === "stage" && + [400, 401, 403, 404, 405, 413].includes(error.status) + ) +} + +function normalizedMigrationID(value: string | undefined): string { + const migrationId = value ?? "legacy-migration" + if (!migrationIdPattern.test(migrationId)) { + throw new Error("hosted migration id is invalid") + } + return migrationId +} + +function migrationTenantKey(value: unknown): string { + if (typeof value !== "string" || !tenantKeyPattern.test(value)) { + throw new Error("hosted migration tenant key is invalid") + } + return value +} + +function migrationDestination(value: unknown, allowLoopback: boolean): string { + if (typeof value !== "string") { + throw new Error("hosted migration destination is invalid") + } + let url: URL + try { + url = new URL(value) + } catch { + throw new Error("hosted migration destination is invalid") + } + if ( + url.username || + url.password || + url.pathname !== "/" || + url.search || + url.hash + ) { + throw new Error("hosted migration destination is invalid") + } + const origin = url.origin + if (productionOrigins.has(origin)) return origin + if ( + allowLoopback && + url.protocol === "http:" && + (url.hostname === "localhost" || + url.hostname === "127.0.0.1" || + url.hostname === "[::1]") + ) { + return origin + } + throw new Error("hosted migration destination is not allowed") +} + +function migrationUpload(account: LegacyMigrationAccount): Record { + const id = normalizedMigrationText(account.id, 320) + const label = normalizedMigrationText(account.label, 320) + if (!id || !label || !account.enabled || account.hasTotp) { + throw new Error("legacy account cannot be migrated safely") + } + const credentials = account.credentials + if (!credentials) { + throw new Error("legacy account credentials are unavailable") + } + switch (account.kind) { + case "codex_oauth": { + const accessToken = requiredSecret(credentials.accessToken) + const refreshToken = requiredSecret(credentials.refreshToken) + const idToken = requiredSecret(credentials.idToken) + const accountID = requiredSecret(credentials.accountId) + return { + provider: "codex", + accountId: id, + label, + tokens: { accessToken, refreshToken, idToken, accountID }, + } + } + case "openai_apikey": + return { + provider: "openai-apikey", + accountId: id, + label, + apiKey: requiredSecret(credentials.apiKey), + } + case "anthropic_apikey": + return { + provider: "anthropic-apikey", + accountId: id, + label, + apiKey: requiredSecret(credentials.apiKey), + } + case "anthropic_oauth": + throw new Error("legacy Claude OAuth migration is not supported") + } +} + +const terminalControlPattern = /[\p{Cc}\p{Cf}\p{Zl}\p{Zp}]/u + +function normalizedMigrationText( + value: unknown, + maxBytes: number +): string | null { + if (typeof value !== "string") return null + const normalized = value.trim() + if ( + !normalized || + terminalControlPattern.test(normalized) || + new TextEncoder().encode(normalized).byteLength > maxBytes + ) { + return null + } + return normalized +} + +function validatedMigrationUploads( + accounts: ReadonlyArray +): ReadonlyArray> { + if (accounts.length === 0 || accounts.length > maxMigrationAccounts) { + throw new Error("hosted migration account limit exceeded") + } + return accounts.map(migrationUpload) +} + +function normalizedString(value: unknown, maxLength: number): string | null { + if (typeof value !== "string") return null + const normalized = value.trim() + return normalized && normalized.length <= maxLength ? normalized : null +} + +function requiredSecret(value: unknown): string { + const normalized = normalizedString(value, 64 * 1024) + if (!normalized) throw new Error("legacy account credentials are incomplete") + return normalized +} diff --git a/cloudflare/packages/worker/test/tenant-migration.test.ts b/cloudflare/packages/worker/test/tenant-migration.test.ts new file mode 100644 index 000000000..ff562b635 --- /dev/null +++ b/cloudflare/packages/worker/test/tenant-migration.test.ts @@ -0,0 +1,876 @@ +import { afterEach, describe, expect, test } from "bun:test" +import { + migrateLegacyAccountsToHosted, + migrateLegacyTenant, +} from "../src/legacy-migration.ts" +import { + adminToken, + cleanupWorkers, + createTenant, + startWorker, + stopWorker, + waitForCondition, +} from "./helpers/worker.ts" + +const servers: Array<{ stop: () => void }> = [] + +afterEach(async () => { + await cleanupWorkers() + for (const server of servers.splice(0)) server.stop() +}) + +describe("legacy tenant migration", () => { + test("stages credentials without activating them before source finalization", async () => { + const uploads: Array> = [] + const calls: string[] = [] + const migrated = await migrateLegacyAccountsToHosted({ + destinationUrl: "https://sr.cmux.com", + tenantKey: "srt_0123456789abcdef0123456789abcdef", + accounts: ["a", "b"].map((suffix) => ({ + id: `legacy-account-${suffix}`, + kind: "codex_oauth" as const, + label: "Shared account", + enabled: true, + hasTotp: false, + credentials: { + accessToken: `legacy-access-${suffix}`, + refreshToken: `legacy-refresh-${suffix}`, + idToken: `legacy-id-${suffix}`, + accountId: `provider-${suffix}`, + }, + })), + fetch: (async (input, init) => { + expect(String(input)).toBe( + "https://sr.cmux.com/_subrouter/accounts/migration/stage" + ) + expect(new Headers(init?.headers).get("authorization")).toBe( + "Bearer srt_0123456789abcdef0123456789abcdef" + ) + calls.push(new URL(String(input)).pathname) + const body = JSON.parse(String(init?.body)) as Record + uploads.push(...body.accounts) + return Response.json({ + ok: true, + accountIds: body.accounts.map( + (upload: Record) => upload.accountId + ), + }) + }) as typeof fetch, + }) + + expect(migrated).toBe(2) + expect(new Set(uploads.map((upload) => upload.accountId)).size).toBe(2) + expect(uploads.map((upload) => upload.label)).toEqual([ + "Shared account", + "Shared account", + ]) + expect(uploads.map((upload) => upload.tokens?.refreshToken).sort()).toEqual([ + "legacy-refresh-a", + "legacy-refresh-b", + ]) + expect(calls).toEqual(["/_subrouter/accounts/migration/stage"]) + }) + + test("quiesces and removes source credentials only after every upload succeeds", async () => { + const sourceState = { + value: "active" as "active" | "quiesced" | "removed", + } + const accounts = [legacyAccount("a"), legacyAccount("b")] + const migrated = await migrateLegacyTenant({ + destinationUrl: "https://sr.cmux.com", + tenantKey: "srt_0123456789abcdef0123456789abcdef", + finalizeSource: true, + source: { + list: async () => accounts, + begin: async () => { + sourceState.value = "quiesced" + return accounts + }, + complete: async () => { + expect(sourceState.value).toBe("quiesced") + sourceState.value = "removed" + }, + restore: async () => { + sourceState.value = "active" + }, + }, + fetch: hostedMigrationFetch(), + }) + + expect(migrated).toBe(2) + expect(sourceState.value).toBe("removed") + }) + + test("restores source routing when a finalized migration upload fails", async () => { + const sourceState = { + value: "active" as "active" | "quiesced" | "removed", + } + await expect( + migrateLegacyTenant({ + destinationUrl: "https://sr.cmux.com", + tenantKey: "srt_0123456789abcdef0123456789abcdef", + finalizeSource: true, + source: { + list: async () => [legacyAccount("a")], + begin: async () => { + sourceState.value = "quiesced" + return [legacyAccount("a")] + }, + complete: async () => { + sourceState.value = "removed" + }, + restore: async () => { + sourceState.value = "active" + }, + }, + fetch: hostedMigrationFetch({ failStage: true }), + }) + ).rejects.toThrow("destination is unavailable") + expect(sourceState.value).toBe("active") + }) + + test("restores source routing when staging and inactive rollback both fail", async () => { + const sourceState = { + value: "active" as "active" | "quiesced" | "removed", + } + let recoveryPreserved = false + await expect( + migrateLegacyTenant({ + destinationUrl: "https://sr.cmux.com", + tenantKey: "srt_0123456789abcdef0123456789abcdef", + finalizeSource: true, + source: { + list: async () => [legacyAccount("a")], + begin: async () => { + sourceState.value = "quiesced" + return [legacyAccount("a")] + }, + complete: async () => { + sourceState.value = "removed" + }, + restore: async (_accounts, options) => { + sourceState.value = "active" + recoveryPreserved = options.preserveRecovery + }, + }, + fetch: hostedMigrationFetch({ + failStage: true, + failRollback: true, + }), + }) + ).rejects.toThrow("destination is unavailable") + expect(sourceState.value).toBe("active") + expect(recoveryPreserved).toBe(true) + }) + + test("clears recovery after an authoritatively rejected stage", async () => { + const sourceState = { + value: "active" as "active" | "quiesced", + } + const destinationCalls: string[] = [] + let recoveryPreserved = true + await expect( + migrateLegacyTenant({ + destinationUrl: "https://sr.cmux.com", + tenantKey: "srt_0123456789abcdef0123456789abcdef", + finalizeSource: true, + source: { + list: async () => [legacyAccount("a")], + begin: async () => { + sourceState.value = "quiesced" + return [legacyAccount("a")] + }, + complete: async () => {}, + restore: async (_accounts, options) => { + sourceState.value = "active" + recoveryPreserved = options.preserveRecovery + }, + }, + fetch: hostedMigrationFetch({ + rejectAuthorization: true, + calls: destinationCalls, + }), + }) + ).rejects.toThrow("stage failed (401)") + expect(sourceState.value).toBe("active") + expect(recoveryPreserved).toBe(false) + expect(destinationCalls).toEqual([ + "/_subrouter/accounts/migration/stage", + ]) + }) + + test("rejects unsafe migration identities before quiescing the source", async () => { + const invalidAccounts = [ + { ...legacyAccount("control"), label: "unsafe\u001b[31m" }, + { ...legacyAccount("label"), label: "é".repeat(161) }, + { ...legacyAccount("id"), id: "é".repeat(161) }, + ] + for (const account of invalidAccounts) { + let sourceBegan = false + const destinationCalls: string[] = [] + await expect( + migrateLegacyTenant({ + destinationUrl: "https://sr.cmux.com", + tenantKey: "srt_0123456789abcdef0123456789abcdef", + finalizeSource: true, + source: { + list: async () => [account], + begin: async () => { + sourceBegan = true + return [account] + }, + complete: async () => {}, + restore: async () => {}, + }, + fetch: hostedMigrationFetch({ calls: destinationCalls }), + }) + ).rejects.toThrow("legacy account cannot be migrated safely") + expect(sourceBegan).toBe(false) + expect(destinationCalls).toEqual([]) + } + }) + + test("accepts migration identities at the 320-byte UTF-8 boundary", async () => { + const migrated = await migrateLegacyAccountsToHosted({ + destinationUrl: "https://sr.cmux.com", + tenantKey: "srt_0123456789abcdef0123456789abcdef", + accounts: [{ + ...legacyAccount("boundary"), + id: "é".repeat(160), + label: "é".repeat(160), + }], + fetch: hostedMigrationFetch(), + }) + expect(migrated).toBe(1) + }) + + test("restores source routing when quiescing fails after changing state", async () => { + const sourceState = { + value: "active" as "active" | "quiesced", + } + await expect( + migrateLegacyTenant({ + destinationUrl: "https://sr.cmux.com", + tenantKey: "srt_0123456789abcdef0123456789abcdef", + finalizeSource: true, + source: { + list: async () => [legacyAccount("a")], + begin: async () => { + sourceState.value = "quiesced" + throw new Error("alarm update failed") + }, + complete: async () => {}, + restore: async () => { + sourceState.value = "active" + }, + }, + }) + ).rejects.toThrow("alarm update failed") + expect(sourceState.value).toBe("active") + }) + + test("finalizes source rows through the admin route after hosted upload", async () => { + const uploads: Array> = [] + const destinationCalls: string[] = [] + const destination = Bun.serve({ + port: 0, + async fetch(request) { + destinationCalls.push(new URL(request.url).pathname) + return await hostedMigrationResponse(request, { uploads }) + }, + }) + servers.push(destination) + const worker = await startWorker() + const tenant = await createTenant(worker.baseURL, "Finalized team") + const secret = "legacy-finalization-refresh-secret" + const upload = await fetch(`${worker.baseURL}/tenant/accounts`, { + method: "POST", + headers: { + authorization: `Bearer ${tenant.key}`, + "content-type": "application/json", + }, + body: JSON.stringify({ + provider: "codex", + label: "Shared account", + tokens: { + accessToken: "legacy-finalization-access", + refreshToken: secret, + idToken: "legacy-finalization-id", + accountID: "legacy-provider-account", + }, + }), + }) + expect(upload.status).toBe(200) + + const response = await fetch( + `${worker.baseURL}/admin/tenants/${tenant.id}/migrate-hosted`, + { + method: "POST", + headers: { + authorization: `Bearer ${adminToken}`, + "content-type": "application/json", + }, + body: JSON.stringify({ + destinationUrl: destination.url.origin, + tenantKey: "srt_0123456789abcdef0123456789abcdef", + finalizeSource: true, + }), + } + ) + const responseText = await response.text() + expect({ status: response.status, body: responseText }).toEqual({ + status: 200, + body: JSON.stringify({ + ok: true, + migrated: 1, + sourceFinalized: true, + }), + }) + expect(JSON.parse(responseText)).toEqual({ + ok: true, + migrated: 1, + sourceFinalized: true, + }) + expect(responseText).not.toContain(secret) + expect(uploads).toHaveLength(1) + expect(uploads[0]?.label).toBe("Shared account") + expect(destinationCalls).toEqual([ + "/_subrouter/accounts/migration/stage", + "/_subrouter/accounts/migration/activate", + ]) + + const source = await fetch(`${worker.baseURL}/tenant/accounts`, { + headers: { authorization: `Bearer ${tenant.key}` }, + }) + expect(source.status).toBe(200) + const sourceBody = (await source.json()) as { accounts: unknown[] } + expect(sourceBody).toEqual({ accounts: [] }) + }, 60_000) + + test("rejects finalization while preflight staging is in progress", async () => { + let firstStageStarted = false + let releaseFirstStage!: () => void + const firstStageReleased = new Promise((resolve) => { + releaseFirstStage = resolve + }) + let stageCount = 0 + const destination = Bun.serve({ + port: 0, + async fetch(request) { + if (new URL(request.url).pathname.endsWith("/stage")) { + stageCount += 1 + if (stageCount === 1) { + firstStageStarted = true + await firstStageReleased + } + } + return await hostedMigrationResponse(request) + }, + }) + servers.push(destination) + const worker = await startWorker() + const tenant = await createTenant(worker.baseURL, "Serialized team") + await uploadLegacyCodexAccount( + worker.baseURL, + tenant.key, + "Serialized account", + "serialized" + ) + + const preflight = migrateTenant(worker.baseURL, tenant.id, { + destinationUrl: destination.url.origin, + finalizeSource: false, + }) + await waitForCondition( + () => firstStageStarted, + "hosted migration preflight to start" + ) + const concurrentFinalization = await migrateTenant( + worker.baseURL, + tenant.id, + { + destinationUrl: destination.url.origin, + finalizeSource: true, + } + ) + releaseFirstStage() + const preflightResponse = await preflight + + expect(concurrentFinalization.status).toBe(409) + expect(preflightResponse.status).toBe(200) + expect(stageCount).toBe(1) + + const finalization = await migrateTenant(worker.baseURL, tenant.id, { + destinationUrl: destination.url.origin, + finalizeSource: true, + }) + expect(finalization.status).toBe(200) + }, 60_000) + + test("returns a completed receipt without touching Hosted on retry", async () => { + const destinationCalls: string[] = [] + const destination = Bun.serve({ + port: 0, + async fetch(request) { + destinationCalls.push(new URL(request.url).pathname) + return await hostedMigrationResponse(request) + }, + }) + servers.push(destination) + const worker = await startWorker() + const tenant = await createTenant(worker.baseURL, "Completed team") + await uploadLegacyCodexAccount( + worker.baseURL, + tenant.key, + "Completed account", + "completed" + ) + + const first = await migrateTenant(worker.baseURL, tenant.id, { + destinationUrl: destination.url.origin, + finalizeSource: true, + }) + expect(first.status).toBe(200) + const callsAfterSuccess = [...destinationCalls] + + const retry = await migrateTenant(worker.baseURL, tenant.id, { + destinationUrl: destination.url.origin, + finalizeSource: true, + }) + expect(retry.status).toBe(200) + const retryBody = (await retry.json()) as Record + expect(retryBody).toEqual({ + ok: true, + migrated: 1, + sourceFinalized: true, + }) + expect(destinationCalls).toEqual(callsAfterSuccess) + }, 60_000) + + test("rejects a concurrent account mutation after source finalization", async () => { + let destinationStarted = false + let releaseDestination!: () => void + const destinationReleased = new Promise((resolve) => { + releaseDestination = resolve + }) + const destination = Bun.serve({ + port: 0, + async fetch(request) { + if (new URL(request.url).pathname.endsWith("/stage")) { + destinationStarted = true + await destinationReleased + } + return await hostedMigrationResponse(request) + }, + }) + servers.push(destination) + const worker = await startWorker() + const tenant = await createTenant(worker.baseURL, "Concurrent team") + const created = await uploadLegacyCodexAccount( + worker.baseURL, + tenant.key, + "Original account", + "original" + ) + + const migration = migrateTenant(worker.baseURL, tenant.id, { + destinationUrl: destination.url.origin, + finalizeSource: true, + }) + await waitForCondition( + () => destinationStarted, + "hosted migration upload to start" + ) + const mutation = fetch(`${worker.baseURL}/admin/accounts`, { + method: "POST", + headers: { + authorization: `Bearer ${adminToken}`, + "content-type": "application/json", + }, + body: JSON.stringify({ + id: created.id, + orgId: tenant.id, + kind: "codex_oauth", + label: "Concurrent account", + enabled: true, + credentials: codexCredentials("concurrent"), + }), + }) + releaseDestination() + + const [migrationResponse, mutationResponse] = await Promise.all([ + migration, + mutation, + ]) + expect(migrationResponse.status).toBe(200) + expect(mutationResponse.status).toBe(400) + const accounts = await listAdminAccounts( + worker.baseURL, + tenant.id + ) + expect(accounts).toEqual([]) + }, 60_000) + + test("retains recovery binding after restart until matching rollback", async () => { + let destinationStarted = false + let hangStage = true + const destinationCalls: string[] = [] + const destination = Bun.serve({ + port: 0, + async fetch(request) { + const path = new URL(request.url).pathname + destinationCalls.push(path) + if (path.endsWith("/stage") && hangStage) { + destinationStarted = true + return new Promise(() => {}) + } + return await hostedMigrationResponse(request) + }, + }) + servers.push(destination) + const worker = await startWorker() + const tenant = await createTenant(worker.baseURL, "Restarted team") + await uploadLegacyCodexAccount( + worker.baseURL, + tenant.key, + "Restarted account", + "restart" + ) + + const migration = migrateTenant(worker.baseURL, tenant.id, { + destinationUrl: destination.url.origin, + finalizeSource: true, + }).catch(() => null) + await waitForCondition( + () => destinationStarted, + "hosted migration upload to start" + ) + await stopWorker(worker) + await migration + + const restarted = await startWorker({ persistDir: worker.persistDir }) + const accounts = await listAdminAccounts(restarted.baseURL, tenant.id) + expect(accounts).toHaveLength(1) + expect(accounts[0]).toMatchObject({ + label: "Restarted account", + enabled: true, + hasCredentials: true, + }) + + const wrongDestinationCalls: string[] = [] + const wrongDestination = Bun.serve({ + port: 0, + async fetch(request) { + wrongDestinationCalls.push(new URL(request.url).pathname) + return await hostedMigrationResponse(request) + }, + }) + servers.push(wrongDestination) + const wrongRetry = await migrateTenant(restarted.baseURL, tenant.id, { + destinationUrl: wrongDestination.url.origin, + finalizeSource: true, + }) + expect(wrongRetry.status).toBe(409) + expect(wrongDestinationCalls).toEqual([]) + + hangStage = false + const callsBeforeRetry = destinationCalls.length + const retry = await migrateTenant(restarted.baseURL, tenant.id, { + destinationUrl: destination.url.origin, + finalizeSource: true, + }) + expect(retry.status).toBe(200) + expect(destinationCalls.slice(callsBeforeRetry)).toEqual([ + "/_subrouter/accounts/migration/rollback", + "/_subrouter/accounts/migration/stage", + "/_subrouter/accounts/migration/activate", + ]) + }, 60_000) + + test("keeps an ambiguous activation quiesced until rollback is confirmed", async () => { + let activationStarted = false + let hangActivation = true + const destinationCalls: string[] = [] + const destination = Bun.serve({ + port: 0, + async fetch(request) { + const path = new URL(request.url).pathname + destinationCalls.push(path) + if (path.endsWith("/activate") && hangActivation) { + activationStarted = true + return new Promise(() => {}) + } + return await hostedMigrationResponse(request) + }, + }) + servers.push(destination) + const worker = await startWorker() + const tenant = await createTenant(worker.baseURL, "Ambiguous team") + await uploadLegacyCodexAccount( + worker.baseURL, + tenant.key, + "Ambiguous account", + "ambiguous" + ) + + const migration = migrateTenant(worker.baseURL, tenant.id, { + destinationUrl: destination.url.origin, + finalizeSource: true, + }).catch(() => null) + await waitForCondition( + () => activationStarted, + "hosted migration activation to start" + ) + await stopWorker(worker) + await migration + + const restarted = await startWorker({ persistDir: worker.persistDir }) + const quiesced = await listAdminAccounts(restarted.baseURL, tenant.id) + expect(quiesced).toHaveLength(1) + expect(quiesced[0]?.enabled).toBe(false) + + hangActivation = false + const retry = await migrateTenant(restarted.baseURL, tenant.id, { + destinationUrl: destination.url.origin, + finalizeSource: true, + }) + expect(retry.status).toBe(200) + expect(destinationCalls).toContain( + "/_subrouter/accounts/migration/rollback" + ) + expect(await listAdminAccounts(restarted.baseURL, tenant.id)).toEqual([]) + }, 60_000) + + test("rejects recovery against a different Hosted destination", async () => { + let activationStarted = false + const originalDestination = Bun.serve({ + port: 0, + async fetch(request) { + if (new URL(request.url).pathname.endsWith("/activate")) { + activationStarted = true + return new Promise(() => {}) + } + return await hostedMigrationResponse(request) + }, + }) + const wrongDestinationCalls: string[] = [] + const wrongDestination = Bun.serve({ + port: 0, + async fetch(request) { + wrongDestinationCalls.push(new URL(request.url).pathname) + return await hostedMigrationResponse(request) + }, + }) + servers.push(originalDestination, wrongDestination) + const worker = await startWorker() + const tenant = await createTenant(worker.baseURL, "Bound team") + await uploadLegacyCodexAccount( + worker.baseURL, + tenant.key, + "Bound account", + "bound" + ) + + const migration = migrateTenant(worker.baseURL, tenant.id, { + destinationUrl: originalDestination.url.origin, + finalizeSource: true, + }).catch(() => null) + await waitForCondition( + () => activationStarted, + "bound migration activation to start" + ) + await stopWorker(worker) + await migration + + const restarted = await startWorker({ persistDir: worker.persistDir }) + const retry = await migrateTenant(restarted.baseURL, tenant.id, { + destinationUrl: wrongDestination.url.origin, + finalizeSource: true, + }) + expect(retry.status).toBe(409) + expect(wrongDestinationCalls).toEqual([]) + const accounts = await listAdminAccounts(restarted.baseURL, tenant.id) + expect(accounts).toHaveLength(1) + expect(accounts[0]?.enabled).toBe(false) + }, 60_000) + + test("admin route rejects untrusted destinations without exposing source credentials", async () => { + const worker = await startWorker() + const tenant = await createTenant(worker.baseURL, "Legacy team") + const secret = "legacy-refresh-secret" + const upload = await fetch(`${worker.baseURL}/tenant/accounts`, { + method: "POST", + headers: { + authorization: `Bearer ${tenant.key}`, + "content-type": "application/json", + }, + body: JSON.stringify({ + provider: "codex", + label: "Shared account", + tokens: { + accessToken: "legacy-access-secret", + refreshToken: secret, + idToken: "legacy-id-secret", + accountID: "provider-account", + }, + }), + }) + expect(upload.status).toBe(200) + + const response = await fetch( + `${worker.baseURL}/admin/tenants/${tenant.id}/migrate-hosted`, + { + method: "POST", + headers: { + authorization: `Bearer ${adminToken}`, + "content-type": "application/json", + }, + body: JSON.stringify({ + destinationUrl: "https://attacker.example", + tenantKey: "srt_0123456789abcdef0123456789abcdef", + }), + } + ) + expect(response.status).toBe(409) + const responseText = await response.text() + expect(responseText).toContain("destination is not allowed") + expect(responseText).not.toContain(secret) + expect(responseText).not.toContain(tenant.key) + }, 60_000) +}) + +const legacyAccount = (suffix: string) => ({ + id: `legacy-account-${suffix}`, + kind: "codex_oauth" as const, + label: "Shared account", + enabled: true, + hasTotp: false, + credentials: { + accessToken: `legacy-access-${suffix}`, + refreshToken: `legacy-refresh-${suffix}`, + idToken: `legacy-id-${suffix}`, + accountId: `provider-${suffix}`, + }, +}) + +const codexCredentials = (suffix: string) => ({ + accessToken: `legacy-access-${suffix}`, + refreshToken: `legacy-refresh-${suffix}`, + idToken: `legacy-id-${suffix}`, + accountId: `provider-${suffix}`, +}) + +const uploadLegacyCodexAccount = async ( + baseURL: string, + tenantKey: string, + label: string, + suffix: string +): Promise<{ id: string }> => { + const response = await fetch(`${baseURL}/tenant/accounts`, { + method: "POST", + headers: { + authorization: `Bearer ${tenantKey}`, + "content-type": "application/json", + }, + body: JSON.stringify({ + provider: "codex", + label, + tokens: { + ...codexCredentials(suffix), + accountID: `provider-${suffix}`, + }, + }), + }) + expect(response.status).toBe(200) + return (await response.json()) as { id: string } +} + +const migrateTenant = async ( + baseURL: string, + tenantId: string, + input: { + readonly destinationUrl: string + readonly finalizeSource: boolean + } +): Promise => + await fetch(`${baseURL}/admin/tenants/${tenantId}/migrate-hosted`, { + method: "POST", + headers: { + authorization: `Bearer ${adminToken}`, + "content-type": "application/json", + }, + body: JSON.stringify({ + ...input, + tenantKey: "srt_0123456789abcdef0123456789abcdef", + }), + }) + +const listAdminAccounts = async ( + baseURL: string, + tenantId: string +): Promise>> => { + const response = await fetch( + `${baseURL}/admin/accounts?tenant=${encodeURIComponent(tenantId)}`, + { headers: { authorization: `Bearer ${adminToken}` } } + ) + expect(response.status).toBe(200) + const body = (await response.json()) as { + accounts: Array> + } + return body.accounts +} + +const hostedMigrationFetch = ( + options: { + readonly failStage?: boolean + readonly failRollback?: boolean + readonly rejectAuthorization?: boolean + readonly calls?: string[] + } = {} +): typeof fetch => + Object.assign( + async (input: Parameters[0], init?: Parameters[1]) => { + const request = new Request(String(input), init) + options.calls?.push(new URL(request.url).pathname) + if (options.rejectAuthorization) { + return new Response("unauthorized", { status: 401 }) + } + if ( + options.failStage && + new URL(request.url).pathname.endsWith("/stage") + ) { + throw new Error("network body with secret") + } + if ( + options.failRollback && + new URL(request.url).pathname.endsWith("/rollback") + ) { + throw new Error("rollback is unavailable") + } + return await hostedMigrationResponse(request) + }, + { preconnect() {} } + ) as typeof fetch + +const hostedMigrationResponse = async ( + request: Request, + options: { readonly uploads?: Array> } = {} +): Promise => { + const path = new URL(request.url).pathname + const body = (await request.json()) as Record + if (path.endsWith("/stage")) { + const accounts = body.accounts as Array> + options.uploads?.push(...accounts) + return Response.json({ + ok: true, + accountIds: accounts.map((account) => account.accountId), + }) + } + if (path.endsWith("/activate")) { + return Response.json({ ok: true, activated: body.accountIds }) + } + if (path.endsWith("/rollback")) { + return Response.json({ ok: true, rolledBack: true }) + } + return new Response("not found", { status: 404 }) +} diff --git a/internal/accounts/codex_auth.go b/internal/accounts/codex_auth.go index f4d684ebd..49b9bb176 100644 --- a/internal/accounts/codex_auth.go +++ b/internal/accounts/codex_auth.go @@ -112,7 +112,7 @@ func (s CodexStore) SyncActiveToStore() error { return err } defer lock.Close() - account, found, err := s.FindStored(email) + account, found, err := s.findStoredExact(email) if err != nil || !found { return err } @@ -218,7 +218,7 @@ func (s CodexStore) refreshStored(ctx context.Context, client *http.Client, acco } defer lock.Close() - latest, found, err := s.FindStored(account.Email) + latest, found, err := s.findStoredExact(account.Email) if err != nil { logCodexRefreshFailed(ctx, s, account, force, err) return account, false, err @@ -277,7 +277,7 @@ func (s CodexStore) refreshStored(ctx context.Context, client *http.Client, acco } func (s CodexStore) recoverRefreshedAccount(previous StoredCodexAccount) (StoredCodexAccount, bool) { - latest, found, err := s.FindStored(previous.Email) + latest, found, err := s.findStoredExact(previous.Email) if err != nil || !found || latest.Auth.Tokens == nil { return StoredCodexAccount{}, false } diff --git a/internal/accounts/codex_store.go b/internal/accounts/codex_store.go index cac781124..cd41d063a 100644 --- a/internal/accounts/codex_store.go +++ b/internal/accounts/codex_store.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "path/filepath" + "slices" "sort" "strings" "time" @@ -17,6 +18,8 @@ type CodexStore struct { Dir string } +const migrationBatchControlLockID = ".subrouter-migration-batch-control" + type StorageKeyCollisionError struct { Identifier string ExistingIdentifier string @@ -27,14 +30,16 @@ func (e *StorageKeyCollisionError) Error() string { } type StoredCodexAccount struct { - Email string `json:"email"` - Provider Provider `json:"provider,omitempty"` - AddedAt string `json:"addedAt"` - Auth CodexAuthFile `json:"auth"` - ProjectID string `json:"projectId,omitempty"` - ProjectName string `json:"projectName,omitempty"` - AdminKeyLabel string `json:"adminKeyLabel,omitempty"` - Breadcrumbs []CodexAuthBreadcrumb `json:"breadcrumbs,omitempty"` + Email string `json:"email"` + Label string `json:"label,omitempty"` + Provider Provider `json:"provider,omitempty"` + MigrationBatchID string `json:"migrationBatchId,omitempty"` + AddedAt string `json:"addedAt"` + Auth CodexAuthFile `json:"auth"` + ProjectID string `json:"projectId,omitempty"` + ProjectName string `json:"projectName,omitempty"` + AdminKeyLabel string `json:"adminKeyLabel,omitempty"` + Breadcrumbs []CodexAuthBreadcrumb `json:"breadcrumbs,omitempty"` } type CodexAuthFile struct { @@ -124,6 +129,23 @@ func (s CodexStore) List() ([]Account, error) { } func (s CodexStore) ListStored() ([]StoredCodexAccount, error) { + lock, err := s.lockStoredAccount(migrationBatchControlLockID) + if err != nil { + return nil, err + } + defer lock.Close() + return s.listStored(false) +} + +func (s CodexStore) listStored(includeInactiveMigrations bool) ([]StoredCodexAccount, error) { + var activeMigrationBatches map[string]struct{} + if !includeInactiveMigrations { + var err error + activeMigrationBatches, err = s.activeMigrationBatchSnapshot() + if err != nil { + return nil, err + } + } files, err := os.ReadDir(s.Dir) if errors.Is(err, os.ErrNotExist) { return nil, nil @@ -148,6 +170,14 @@ func (s CodexStore) ListStored() ([]StoredCodexAccount, error) { if err := json.Unmarshal(body, &stored); err != nil { return nil, fmt.Errorf("parse %s: %w", path, err) } + if !includeInactiveMigrations && stored.MigrationBatchID != "" { + if err := validateMigrationBatchID(stored.MigrationBatchID); err != nil { + return nil, err + } + if _, active := activeMigrationBatches[stored.MigrationBatchID]; !active { + continue + } + } if strings.TrimSpace(stored.Email) != "" { accounts = append(accounts, stored) } @@ -185,10 +215,14 @@ func (a StoredCodexAccount) toAccount(source string) (Account, bool) { } addedAt, _ := time.Parse(time.RFC3339, a.AddedAt) + label := strings.TrimSpace(a.Label) + if label == "" { + label = id + } out := Account{ ID: id, Provider: a.ProviderOrDefault(), - Label: id, + Label: label, Email: id, AddedAt: addedAt, Source: source, @@ -218,6 +252,13 @@ func (s CodexStore) SaveStored(account StoredCodexAccount) error { if err := validateStoredAccountIdentifier(account.Email); err != nil { return err } + if account.MigrationBatchID != "" { + batchLock, err := s.lockStoredAccount(migrationBatchControlLockID) + if err != nil { + return err + } + defer batchLock.Close() + } lock, err := s.lockStoredAccount(account.Email) if err != nil { return err @@ -230,7 +271,7 @@ func (s CodexStore) saveStoredUnlocked(account StoredCodexAccount) error { if err := validateStoredAccountIdentifier(account.Email); err != nil { return err } - stored, err := s.ListStored() + stored, err := s.listStored(true) if err != nil { return err } @@ -259,6 +300,19 @@ func (s CodexStore) saveStoredUnlocked(account StoredCodexAccount) error { ExistingIdentifier: existing.Email, } } + if existing.MigrationBatchID != account.MigrationBatchID { + if account.MigrationBatchID != "" { + return fmt.Errorf("account %q belongs to a different migration state", account.Email) + } + active, activeErr := s.migrationBatchActive(existing.MigrationBatchID) + if activeErr != nil { + return activeErr + } + if existing.MigrationBatchID == "" || !active { + return fmt.Errorf("account %q belongs to a different migration state", account.Email) + } + account.MigrationBatchID = existing.MigrationBatchID + } } else if !errors.Is(err, os.ErrNotExist) { return err } @@ -292,21 +346,12 @@ func (s CodexStore) FindStored(identifier string) (StoredCodexAccount, bool, err if needle == "" { return StoredCodexAccount{}, false, nil } - directPath := filepath.Join(s.Dir, emailToFilename(needle)) - if body, err := os.ReadFile(directPath); err == nil { - var account StoredCodexAccount - if err := json.Unmarshal(body, &account); err != nil { - return StoredCodexAccount{}, false, err - } - if strings.EqualFold(strings.TrimSpace(account.Email), needle) { - return account, true, nil - } - } else if !errors.Is(err, os.ErrNotExist) { - return StoredCodexAccount{}, false, err + if account, ok, err := s.findStoredExact(needle); err != nil || ok { + return account, ok, err } if !strings.HasPrefix(needle, "apikey:") && !strings.Contains(needle, "@") { - if account, ok, err := s.FindStored("apikey:" + needle); err != nil || ok { + if account, ok, err := s.findStoredExact("apikey:" + needle); err != nil || ok { return account, ok, err } } @@ -335,11 +380,305 @@ func (s CodexStore) FindStored(identifier string) (StoredCodexAccount, bool, err return matches[0], true, nil } +func (s CodexStore) findStoredExact(identifier string) (StoredCodexAccount, bool, error) { + needle := strings.TrimSpace(identifier) + if needle == "" { + return StoredCodexAccount{}, false, nil + } + directPath := filepath.Join(s.Dir, emailToFilename(needle)) + if body, err := os.ReadFile(directPath); err == nil { + var account StoredCodexAccount + if err := json.Unmarshal(body, &account); err != nil { + return StoredCodexAccount{}, false, err + } + if strings.EqualFold(strings.TrimSpace(account.Email), needle) { + if account.MigrationBatchID != "" { + active, activeErr := s.migrationBatchActive(account.MigrationBatchID) + if activeErr != nil { + return StoredCodexAccount{}, false, activeErr + } + if !active { + return StoredCodexAccount{}, false, nil + } + } + return account, true, nil + } + } else if !errors.Is(err, os.ErrNotExist) { + return StoredCodexAccount{}, false, err + } + + return StoredCodexAccount{}, false, nil +} + +// StageMigrationBatch persists credentials outside every routing and refresh +// path. A single activation marker later makes the complete batch visible. +func (s CodexStore) StageMigrationBatch(batchID string, staged []StoredCodexAccount) error { + if err := validateMigrationBatchID(batchID); err != nil { + return err + } + lock, err := s.lockStoredAccount(migrationBatchControlLockID) + if err != nil { + return err + } + defer lock.Close() + if active, err := s.migrationBatchActive(batchID); err != nil { + return err + } else if active { + return errors.New("migration batch is already active") + } + desired := make(map[string]bool, len(staged)) + accountIDs := make([]string, 0, len(staged)) + for i := range staged { + if err := validateStoredAccountIdentifier(staged[i].Email); err != nil { + return err + } + key := strings.ToLower(strings.TrimSpace(staged[i].Email)) + if desired[key] { + return fmt.Errorf("duplicate migration account %q", staged[i].Email) + } + desired[key] = true + staged[i].MigrationBatchID = batchID + accountIDs = append(accountIDs, staged[i].Email) + } + all, err := s.listStored(true) + if err != nil { + return err + } + for _, account := range all { + if account.MigrationBatchID == batchID { + accountIDs = append(accountIDs, account.Email) + } + } + accountLocks, err := s.lockStoredAccounts(accountIDs) + if err != nil { + return err + } + defer closeAccountFileLocks(accountLocks) + for i := range staged { + if err := s.saveStoredUnlocked(staged[i]); err != nil { + return err + } + } + all, err = s.listStored(true) + if err != nil { + return err + } + for _, account := range all { + if account.MigrationBatchID != batchID || desired[strings.ToLower(strings.TrimSpace(account.Email))] { + continue + } + if err := os.Remove(account.SourcePath(s)); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + } + return nil +} + +// ActivateMigrationBatch atomically publishes a previously staged batch. +func (s CodexStore) ActivateMigrationBatch(batchID string, expectedIDs []string) error { + if err := validateMigrationBatchID(batchID); err != nil { + return err + } + lock, err := s.lockStoredAccount(migrationBatchControlLockID) + if err != nil { + return err + } + defer lock.Close() + all, err := s.listStored(true) + if err != nil { + return err + } + actual := make([]string, 0, len(expectedIDs)) + for _, account := range all { + if account.MigrationBatchID == batchID { + actual = append(actual, account.Email) + } + } + sort.Strings(actual) + expected := append([]string(nil), expectedIDs...) + for i := range expected { + expected[i] = strings.TrimSpace(expected[i]) + if err := validateStoredAccountIdentifier(expected[i]); err != nil { + return err + } + } + sort.Strings(expected) + accountLocks, err := s.lockStoredAccounts(append(append([]string(nil), actual...), expected...)) + if err != nil { + return err + } + defer closeAccountFileLocks(accountLocks) + all, err = s.listStored(true) + if err != nil { + return err + } + actual = actual[:0] + for _, account := range all { + if account.MigrationBatchID == batchID { + actual = append(actual, account.Email) + } + } + sort.Strings(actual) + if !slices.Equal(actual, expected) { + return errors.New("migration batch account set does not match staged credentials") + } + body, err := json.Marshal(map[string]any{"accountIds": expected}) + if err != nil { + return err + } + return writeFileAtomic(s.migrationBatchMarker(batchID), append(body, '\n'), 0o600) +} + +// RollbackMigrationBatch deactivates and removes only credentials owned by the +// named batch. It is idempotent so callers can resolve an ambiguous activation. +func (s CodexStore) RollbackMigrationBatch(batchID string) error { + if err := validateMigrationBatchID(batchID); err != nil { + return err + } + lock, err := s.lockStoredAccount(migrationBatchControlLockID) + if err != nil { + return err + } + defer lock.Close() + all, err := s.listStored(true) + if err != nil { + return err + } + accountIDs := make([]string, 0, len(all)) + for _, account := range all { + if account.MigrationBatchID == batchID { + accountIDs = append(accountIDs, account.Email) + } + } + accountLocks, err := s.lockStoredAccounts(accountIDs) + if err != nil { + return err + } + defer closeAccountFileLocks(accountLocks) + if err := os.Remove(s.migrationBatchMarker(batchID)); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + all, err = s.listStored(true) + if err != nil { + return err + } + for _, account := range all { + if account.MigrationBatchID != batchID { + continue + } + if err := os.Remove(account.SourcePath(s)); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + } + return nil +} + +func validateMigrationBatchID(batchID string) error { + if batchID == "" || len(batchID) > 160 { + return errors.New("migration id is invalid") + } + for _, character := range batchID { + if (character >= 'a' && character <= 'z') || + (character >= '0' && character <= '9') || + character == '-' || character == '_' || character == '.' { + continue + } + return errors.New("migration id is invalid") + } + return nil +} + +func (s CodexStore) lockStoredAccounts(identifiers []string) ([]*accountFileLock, error) { + unique := make(map[string]struct{}, len(identifiers)) + canonical := make([]string, 0, len(identifiers)) + for _, identifier := range identifiers { + key := strings.ToLower(strings.TrimSpace(identifier)) + if key == "" { + continue + } + if _, exists := unique[key]; exists { + continue + } + unique[key] = struct{}{} + canonical = append(canonical, key) + } + sort.Strings(canonical) + locks := make([]*accountFileLock, 0, len(canonical)) + for _, identifier := range canonical { + lock, err := s.lockStoredAccount(identifier) + if err != nil { + closeAccountFileLocks(locks) + return nil, err + } + locks = append(locks, lock) + } + return locks, nil +} + +func closeAccountFileLocks(locks []*accountFileLock) { + for i := len(locks) - 1; i >= 0; i-- { + _ = locks[i].Close() + } +} + +func (s CodexStore) migrationBatchMarker(batchID string) string { + return filepath.Join(s.Dir, ".migration-batches", batchID+".active.json") +} + +func (s CodexStore) activeMigrationBatchSnapshot() (map[string]struct{}, error) { + active := make(map[string]struct{}) + entries, err := os.ReadDir(filepath.Join(s.Dir, ".migration-batches")) + if errors.Is(err, os.ErrNotExist) { + return active, nil + } + if err != nil { + return nil, err + } + const suffix = ".active.json" + for _, entry := range entries { + if entry.IsDir() || !strings.HasSuffix(entry.Name(), suffix) { + continue + } + batchID := strings.TrimSuffix(entry.Name(), suffix) + if err := validateMigrationBatchID(batchID); err != nil { + continue + } + active[batchID] = struct{}{} + } + return active, nil +} + +func (s CodexStore) migrationBatchActive(batchID string) (bool, error) { + if batchID == "" { + return false, nil + } + if err := validateMigrationBatchID(batchID); err != nil { + return false, err + } + _, err := os.Stat(s.migrationBatchMarker(batchID)) + if err == nil { + return true, nil + } + if errors.Is(err, os.ErrNotExist) { + return false, nil + } + return false, err +} + func (s CodexStore) RemoveStored(identifier string) (StoredCodexAccount, bool, error) { account, ok, err := s.FindStored(identifier) if err != nil || !ok { return account, ok, err } + lock, err := s.lockStoredAccount(account.Email) + if err != nil { + return account, false, err + } + defer lock.Close() + account, ok, err = s.findStoredExact(account.Email) + if err != nil || !ok { + return account, ok, err + } if err := os.Remove(filepath.Join(s.Dir, emailToFilename(account.Email))); err != nil { return account, false, err } @@ -360,6 +699,15 @@ func (s CodexStore) MigrateStoredAway(identifier string) (string, bool, error) { if err != nil || !ok { return "", ok, err } + lock, err := s.lockStoredAccount(account.Email) + if err != nil { + return "", false, err + } + defer lock.Close() + account, ok, err = s.findStoredExact(account.Email) + if err != nil || !ok { + return "", ok, err + } name := emailToFilename(account.Email) dest := filepath.Join(s.Dir, MigratedDirName) if err := os.MkdirAll(dest, 0o700); err != nil { diff --git a/internal/accounts/codex_store_test.go b/internal/accounts/codex_store_test.go index fe75057c5..f49fc2dee 100644 --- a/internal/accounts/codex_store_test.go +++ b/internal/accounts/codex_store_test.go @@ -1,10 +1,180 @@ package accounts import ( + "fmt" "os" "testing" + "time" ) +func TestCodexStoreMigrationBatchVisibilityIsAtomicForConcurrentReaders(t *testing.T) { + store := CodexStore{Dir: t.TempDir()} + const accountCount = 64 + staged := make([]StoredCodexAccount, 0, accountCount) + for i := 0; i < accountCount; i++ { + id := fmt.Sprintf("migration-%03d@example.com", i) + staged = append(staged, StoredCodexAccount{ + Email: id, + Provider: ProviderCodex, + Auth: CodexAuthFile{ + AuthMode: "apikey", + OpenAIAPIKey: "sk-test", + }, + }) + } + const batchID = "atomic-visibility" + if err := store.StageMigrationBatch(batchID, staged); err != nil { + t.Fatal(err) + } + + stop := make(chan struct{}) + done := make(chan struct{}) + errors := make(chan error, 1) + go func() { + defer close(done) + marker := store.migrationBatchMarker(batchID) + body := []byte(`{"accountIds":[]}` + "\n") + for { + select { + case <-stop: + return + default: + } + if err := writeFileAtomic(marker, body, 0o600); err != nil { + errors <- err + return + } + if err := os.Remove(marker); err != nil { + errors <- err + return + } + } + }() + defer func() { + close(stop) + <-done + }() + + for i := 0; i < 500; i++ { + select { + case err := <-errors: + t.Fatal(err) + default: + } + accounts, err := store.ListStored() + if err != nil { + t.Fatal(err) + } + if len(accounts) != 0 && len(accounts) != accountCount { + t.Fatalf("concurrent batch list exposed %d of %d accounts", len(accounts), accountCount) + } + } +} + +func TestCodexStoreRollbackOwnsAnOrdinaryReplacementOfAnActiveBatchAccount(t *testing.T) { + store := CodexStore{Dir: t.TempDir()} + staged := StoredCodexAccount{ + Email: "migrated@example.com", + Provider: ProviderCodex, + Auth: CodexAuthFile{ + AuthMode: "apikey", + OpenAIAPIKey: "sk-migrated", + }, + } + const batchID = "replacement-ownership" + if err := store.StageMigrationBatch(batchID, []StoredCodexAccount{staged}); err != nil { + t.Fatal(err) + } + if err := store.ActivateMigrationBatch(batchID, []string{staged.Email}); err != nil { + t.Fatal(err) + } + replacement := staged + replacement.MigrationBatchID = "" + replacement.Auth.OpenAIAPIKey = "sk-repaired" + if err := store.SaveStored(replacement); err != nil { + t.Fatal(err) + } + if err := store.RollbackMigrationBatch(batchID); err != nil { + t.Fatal(err) + } + accounts, err := store.ListStored() + if err != nil { + t.Fatal(err) + } + if len(accounts) != 0 { + t.Fatalf("rollback left ordinary replacement active: %+v", accounts) + } +} + +func TestCodexStoreRollbackIsAtomicForConcurrentReaders(t *testing.T) { + store := CodexStore{Dir: t.TempDir()} + const accountCount = 24 + const batchID = "rollback-visibility" + staged := make([]StoredCodexAccount, 0, accountCount) + ids := make([]string, 0, accountCount) + for i := 0; i < accountCount; i++ { + id := fmt.Sprintf("rollback-%03d@example.com", i) + ids = append(ids, id) + staged = append(staged, StoredCodexAccount{ + Email: id, + Provider: ProviderCodex, + Auth: CodexAuthFile{ + AuthMode: "apikey", + OpenAIAPIKey: "sk-test", + }, + }) + } + if err := store.StageMigrationBatch(batchID, staged); err != nil { + t.Fatal(err) + } + + stop := make(chan struct{}) + done := make(chan struct{}) + errors := make(chan error, 1) + go func() { + defer close(done) + for { + select { + case <-stop: + return + default: + } + if err := store.ActivateMigrationBatch(batchID, ids); err != nil { + errors <- err + return + } + if err := store.RollbackMigrationBatch(batchID); err != nil { + errors <- err + return + } + if err := store.StageMigrationBatch(batchID, staged); err != nil { + errors <- err + return + } + time.Sleep(time.Millisecond) + } + }() + defer func() { + close(stop) + <-done + }() + + for i := 0; i < 200; i++ { + select { + case err := <-errors: + t.Fatal(err) + default: + } + accounts, err := store.ListStored() + if err != nil { + t.Fatal(err) + } + if len(accounts) != 0 && len(accounts) != accountCount { + t.Fatalf("concurrent rollback list exposed %d of %d accounts", len(accounts), accountCount) + } + } +} + func TestCodexStoreRejectsAccountIdentifierThatWouldCreateHiddenState(t *testing.T) { store := CodexStore{Dir: t.TempDir()} account := StoredCodexAccount{ diff --git a/internal/broker/client.go b/internal/broker/client.go index 7d927da31..9ec1db9fd 100644 --- a/internal/broker/client.go +++ b/internal/broker/client.go @@ -199,6 +199,7 @@ func (c *Client) ListAccounts(ctx context.Context) ([]SharedAccount, error) { ID string `json:"id"` Provider string `json:"provider"` AuthMode string `json:"auth_mode"` + Label string `json:"label"` Email string `json:"email"` Health *struct { OK bool `json:"ok"` @@ -221,7 +222,10 @@ func (c *Client) ListAccounts(ctx context.Context) ([]SharedAccount, error) { return nil, fmt.Errorf("unsupported hosted account provider %q", item.Provider) } } - label := item.Email + label := item.Label + if label == "" { + label = item.Email + } if label == "" { label = item.ID } diff --git a/internal/broker/hosted_client_test.go b/internal/broker/hosted_client_test.go index cd648d562..4937c8230 100644 --- a/internal/broker/hosted_client_test.go +++ b/internal/broker/hosted_client_test.go @@ -24,6 +24,7 @@ func TestHostedClientUsesTenantScopedDirectAccountAPI(t *testing.T) { _ = json.NewEncoder(w).Encode([]map[string]any{{ "id": "user@example.com", "provider": "codex", "auth_mode": "oauth", "email": "user@example.com", + "label": "Shared Codex", "health": map[string]any{"ok": false, "message": "refresh failed"}, }}) case http.MethodPost: @@ -47,7 +48,7 @@ func TestHostedClientUsesTenantScopedDirectAccountAPI(t *testing.T) { if err != nil { t.Fatal(err) } - if len(items) != 1 || items[0].Kind != "codex" { + if len(items) != 1 || items[0].Kind != "codex" || items[0].Label != "Shared Codex" { t.Fatalf("items = %#v", items) } if items[0].Health == nil || items[0].Health.OK || diff --git a/internal/proxy/multitenant.go b/internal/proxy/multitenant.go index 1206c2d45..e4dd4aba3 100644 --- a/internal/proxy/multitenant.go +++ b/internal/proxy/multitenant.go @@ -323,6 +323,18 @@ func tenantScopedHandler(server Server, t tenant.Tenant) http.Handler { return } } + if r.URL.Path == "/_subrouter/accounts/migration/stage" && r.Method == http.MethodPost { + handleTenantMigrationStage(&server, w, r) + return + } + if r.URL.Path == "/_subrouter/accounts/migration/activate" && r.Method == http.MethodPost { + handleTenantMigrationActivate(&server, w, r) + return + } + if r.URL.Path == "/_subrouter/accounts/migration/rollback" && r.Method == http.MethodPost { + handleTenantMigrationRollback(&server, w, r) + return + } if strings.HasPrefix(r.URL.Path, "/_subrouter/accounts/") && r.Method == http.MethodDelete { handleTenantAccountDelete(&server, w, r) return @@ -674,6 +686,7 @@ func validStackTeamName(name string) bool { type tenantAccountUpload struct { Provider string `json:"provider"` + AccountID string `json:"accountId,omitempty"` Label string `json:"label"` APIKey string `json:"apiKey"` TargetAccountID string `json:"targetAccountID,omitempty"` @@ -686,6 +699,150 @@ type tenantAccountUpload struct { ClaudeAIOAuth *agentclaude.CredentialInfo `json:"claudeAiOauth"` } +type tenantMigrationStageInput struct { + MigrationID string `json:"migrationId"` + Accounts []tenantAccountUpload `json:"accounts"` +} + +type tenantMigrationBatchInput struct { + MigrationID string `json:"migrationId"` + AccountIDs []string `json:"accountIds,omitempty"` +} + +func handleTenantMigrationStage(server *Server, w http.ResponseWriter, r *http.Request) { + if server.AccountRef == nil { + http.Error(w, "tenant account store unavailable", http.StatusServiceUnavailable) + return + } + var input tenantMigrationStageInput + if !decodeTenantMigrationJSON(server, w, r, &input) { + return + } + if len(input.Accounts) == 0 || len(input.Accounts) > 16 { + http.Error(w, "migration account count is invalid", http.StatusBadRequest) + return + } + staged := make([]accounts.StoredCodexAccount, 0, len(input.Accounts)) + ids := make([]string, 0, len(input.Accounts)) + for _, upload := range input.Accounts { + account, err := storedTenantMigrationAccount(upload) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + staged = append(staged, account) + ids = append(ids, account.Email) + } + if err := server.AccountRef.store.StageMigrationBatch(strings.TrimSpace(input.MigrationID), staged); err != nil { + http.Error(w, "stage migration accounts", http.StatusConflict) + return + } + writeJSON(w, map[string]any{"ok": true, "accountIds": ids}) +} + +func handleTenantMigrationActivate(server *Server, w http.ResponseWriter, r *http.Request) { + if server.AccountRef == nil { + http.Error(w, "tenant account store unavailable", http.StatusServiceUnavailable) + return + } + var input tenantMigrationBatchInput + if !decodeTenantMigrationJSON(server, w, r, &input) { + return + } + batchID := strings.TrimSpace(input.MigrationID) + if err := server.AccountRef.store.ActivateMigrationBatch(batchID, input.AccountIDs); err != nil { + http.Error(w, "activate migration accounts", http.StatusConflict) + return + } + if _, _, err := server.reloadAccounts(r.Context()); err != nil { + _ = server.AccountRef.store.RollbackMigrationBatch(batchID) + _, _, _ = server.reloadAccounts(r.Context()) + http.Error(w, "activate migration accounts", http.StatusInternalServerError) + return + } + writeJSON(w, map[string]any{"ok": true, "activated": input.AccountIDs}) +} + +func handleTenantMigrationRollback(server *Server, w http.ResponseWriter, r *http.Request) { + if server.AccountRef == nil { + http.Error(w, "tenant account store unavailable", http.StatusServiceUnavailable) + return + } + var input tenantMigrationBatchInput + if !decodeTenantMigrationJSON(server, w, r, &input) { + return + } + if err := server.AccountRef.store.RollbackMigrationBatch(strings.TrimSpace(input.MigrationID)); err != nil { + http.Error(w, "rollback migration accounts", http.StatusInternalServerError) + return + } + if _, _, err := server.reloadAccounts(r.Context()); err != nil { + http.Error(w, "rollback migration accounts", http.StatusInternalServerError) + return + } + writeJSON(w, map[string]any{"ok": true, "rolledBack": true}) +} + +func decodeTenantMigrationJSON(server *Server, w http.ResponseWriter, r *http.Request, output any) bool { + bodyLimit := server.MaxBodyBytes + if bodyLimit <= 0 { + bodyLimit = 1 << 20 + } + decoder := json.NewDecoder(io.LimitReader(r.Body, bodyLimit)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(output); err != nil { + http.Error(w, "invalid JSON body", http.StatusBadRequest) + return false + } + if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) { + http.Error(w, "invalid JSON body", http.StatusBadRequest) + return false + } + return true +} + +func storedTenantMigrationAccount(input tenantAccountUpload) (accounts.StoredCodexAccount, error) { + input.Provider = strings.ToLower(strings.TrimSpace(input.Provider)) + input.AccountID = strings.TrimSpace(input.AccountID) + input.Label = strings.TrimSpace(input.Label) + if !validTenantAccountText(input.Label) { + return accounts.StoredCodexAccount{}, errors.New("account label is invalid") + } + if !validTenantAccountText(input.AccountID) { + return accounts.StoredCodexAccount{}, errors.New("account id is invalid") + } + switch input.Provider { + case "codex": + if input.Tokens == nil || input.Tokens.AccessToken == "" || input.Tokens.RefreshToken == "" || input.Tokens.IDToken == "" { + return accounts.StoredCodexAccount{}, errors.New("complete Codex OAuth tokens are required") + } + return accounts.StoredCodexAccount{ + Email: input.AccountID, Label: input.Label, Provider: accounts.ProviderCodex, + Auth: accounts.CodexAuthFile{ + AuthMode: "chatgpt", + Tokens: &accounts.CodexTokens{ + AccessToken: input.Tokens.AccessToken, RefreshToken: input.Tokens.RefreshToken, + IDToken: input.Tokens.IDToken, AccountID: input.Tokens.AccountID, + }, + }, + }, nil + case "openai-apikey", "anthropic-apikey": + if strings.TrimSpace(input.APIKey) == "" { + return accounts.StoredCodexAccount{}, errors.New("API key is required") + } + provider := accounts.ProviderCodex + if input.Provider == "anthropic-apikey" { + provider = accounts.ProviderClaude + } + return accounts.StoredCodexAccount{ + Email: input.AccountID, Label: input.Label, Provider: provider, + Auth: accounts.CodexAuthFile{AuthMode: "apikey", OpenAIAPIKey: strings.TrimSpace(input.APIKey)}, + }, nil + default: + return accounts.StoredCodexAccount{}, errors.New("unsupported migration provider") + } +} + func handleTenantAccountUpload(server *Server, w http.ResponseWriter, r *http.Request) { if server.AccountRef == nil { http.Error(w, "tenant account store unavailable", http.StatusServiceUnavailable) @@ -707,10 +864,16 @@ func handleTenantAccountUpload(server *Server, w http.ResponseWriter, r *http.Re return } input.Provider = strings.ToLower(strings.TrimSpace(input.Provider)) + input.AccountID = strings.TrimSpace(input.AccountID) input.Label = strings.TrimSpace(input.Label) input.TargetAccountID = strings.TrimSpace(input.TargetAccountID) - if input.Label == "" || len(input.Label) > 320 { - http.Error(w, "account label is required", http.StatusBadRequest) + if !validTenantAccountText(input.Label) { + http.Error(w, "account label is invalid", http.StatusBadRequest) + return + } + if input.AccountID != "" && + !validTenantAccountText(input.AccountID) { + http.Error(w, "account id is invalid", http.StatusBadRequest) return } var id string @@ -726,13 +889,16 @@ func handleTenantAccountUpload(server *Server, w http.ResponseWriter, r *http.Re http.Error(w, "complete Codex OAuth tokens are required", http.StatusBadRequest) return } - id, kind = input.Label, "codex" + id, kind = input.AccountID, "codex" + if id == "" { + id = input.Label + } if !validateRepairTarget(id) { http.Error(w, "repair target does not match uploaded account", http.StatusConflict) return } err := server.AccountRef.store.SaveStored(accounts.StoredCodexAccount{ - Email: input.Label, Provider: accounts.ProviderCodex, + Email: id, Label: input.Label, Provider: accounts.ProviderCodex, Auth: accounts.CodexAuthFile{ AuthMode: "chatgpt", Tokens: &accounts.CodexTokens{ @@ -754,13 +920,16 @@ func handleTenantAccountUpload(server *Server, w http.ResponseWriter, r *http.Re if input.Provider == "anthropic-apikey" { provider = accounts.ProviderClaude } - id, kind = "apikey:"+input.Provider+":"+input.Label, input.Provider + id, kind = input.AccountID, input.Provider + if id == "" { + id = "apikey:" + input.Provider + ":" + input.Label + } if !validateRepairTarget(id) { http.Error(w, "repair target does not match uploaded account", http.StatusConflict) return } if err := server.AccountRef.store.SaveStored(accounts.StoredCodexAccount{ - Email: id, Provider: provider, + Email: id, Label: input.Label, Provider: provider, Auth: accounts.CodexAuthFile{AuthMode: "apikey", OpenAIAPIKey: strings.TrimSpace(input.APIKey)}, }); err != nil { http.Error(w, "save API key", http.StatusInternalServerError) @@ -793,6 +962,10 @@ func handleTenantAccountUpload(server *Server, w http.ResponseWriter, r *http.Re }}) } +func validTenantAccountText(value string) bool { + return value != "" && len(value) <= 320 && !containsTerminalControl(value) +} + func handleTenantAccountDelete(server *Server, w http.ResponseWriter, r *http.Request) { if server.AccountRef == nil { http.Error(w, "tenant account store unavailable", http.StatusServiceUnavailable) diff --git a/internal/proxy/multitenant_test.go b/internal/proxy/multitenant_test.go index 0db25a74e..466e1f373 100644 --- a/internal/proxy/multitenant_test.go +++ b/internal/proxy/multitenant_test.go @@ -12,6 +12,7 @@ import ( "net/url" "os" "path/filepath" + "sort" "strings" "sync/atomic" "testing" @@ -1274,6 +1275,189 @@ func TestTenantAccountUploadValidatesRepairTargetAndBodyShape(t *testing.T) { } } +func TestTenantAccountUploadPreservesDistinctMigrationIDsAndLabels(t *testing.T) { + registry, handler, _ := newMultiTenantFixture(t) + _, key, err := registry.Create("team") + if err != nil { + t.Fatal(err) + } + path := "/t/" + key + "/_subrouter/accounts" + for _, id := range []string{"legacy-account-a", "legacy-account-b"} { + body := fmt.Sprintf(`{ + "provider":"codex", + "accountId":%q, + "label":"Shared account", + "tokens":{ + "accessToken":"access-%s", + "refreshToken":"refresh-%s", + "idToken":"id-%s", + "accountID":"provider-%s" + } + }`, id, id, id, id, id) + response := httptest.NewRecorder() + handler.ServeHTTP( + response, + httptest.NewRequest(http.MethodPost, path, strings.NewReader(body)), + ) + if response.Code != http.StatusOK { + t.Fatalf("upload %s status = %d, body = %s", id, response.Code, response.Body.String()) + } + } + + response := httptest.NewRecorder() + handler.ServeHTTP(response, httptest.NewRequest(http.MethodGet, path, nil)) + if response.Code != http.StatusOK { + t.Fatalf("list status = %d, body = %s", response.Code, response.Body.String()) + } + var accounts []struct { + ID string `json:"id"` + Label string `json:"label"` + } + if err := json.Unmarshal(response.Body.Bytes(), &accounts); err != nil { + t.Fatal(err) + } + if len(accounts) != 2 { + t.Fatalf("accounts = %#v, want two distinct migrated accounts", accounts) + } + sort.Slice(accounts, func(i, j int) bool { return accounts[i].ID < accounts[j].ID }) + for i, id := range []string{"legacy-account-a", "legacy-account-b"} { + if accounts[i].ID != id || accounts[i].Label != "Shared account" { + t.Fatalf("account %d = %#v", i, accounts[i]) + } + } +} + +func TestTenantAccountTextRejectsControlsAndUsesUTF8ByteLimits(t *testing.T) { + base := tenantAccountUpload{ + Provider: "openai-apikey", + AccountID: "legacy-account", + Label: "Shared account", + APIKey: "sk-test", + } + for name, mutate := range map[string]func(*tenantAccountUpload){ + "label control": func(input *tenantAccountUpload) { + input.Label = "unsafe\x1b[31m" + }, + "id control": func(input *tenantAccountUpload) { + input.AccountID = "unsafe\naccount" + }, + "label over 320 bytes": func(input *tenantAccountUpload) { + input.Label = strings.Repeat("é", 161) + }, + "id over 320 bytes": func(input *tenantAccountUpload) { + input.AccountID = strings.Repeat("é", 161) + }, + } { + t.Run(name, func(t *testing.T) { + input := base + mutate(&input) + if _, err := storedTenantMigrationAccount(input); err == nil { + t.Fatal("unsafe migration account text was accepted") + } + }) + } + + boundary := base + boundary.AccountID = strings.Repeat("é", 160) + boundary.Label = strings.Repeat("é", 160) + if _, err := storedTenantMigrationAccount(boundary); err != nil { + t.Fatalf("320-byte migration text rejected: %v", err) + } + + registry, handler, _ := newMultiTenantFixture(t) + _, key, err := registry.Create("team") + if err != nil { + t.Fatal(err) + } + body, err := json.Marshal(map[string]string{ + "provider": "openai-apikey", + "accountId": "safe-account", + "label": "unsafe\x1b[31m", + "apiKey": "sk-test", + }) + if err != nil { + t.Fatal(err) + } + response := httptest.NewRecorder() + handler.ServeHTTP( + response, + httptest.NewRequest( + http.MethodPost, + "/t/"+key+"/_subrouter/accounts", + strings.NewReader(string(body)), + ), + ) + if response.Code != http.StatusBadRequest { + t.Fatalf("ordinary upload status = %d, body = %s", response.Code, response.Body.String()) + } +} + +func TestTenantMigrationBatchStaysInactiveUntilAtomicActivation(t *testing.T) { + registry, handler, _ := newMultiTenantFixture(t) + _, key, err := registry.Create("team") + if err != nil { + t.Fatal(err) + } + basePath := "/t/" + key + "/_subrouter/accounts" + request := func(path, body string) *httptest.ResponseRecorder { + t.Helper() + response := httptest.NewRecorder() + handler.ServeHTTP( + response, + httptest.NewRequest(http.MethodPost, path, strings.NewReader(body)), + ) + return response + } + stage := request(basePath+"/migration/stage", `{ + "migrationId":"legacy-team", + "accounts":[{ + "provider":"codex", + "accountId":"legacy-account", + "label":"Shared account", + "tokens":{ + "accessToken":"access", + "refreshToken":"refresh", + "idToken":"id", + "accountID":"provider" + } + }] + }`) + if stage.Code != http.StatusOK { + t.Fatalf("stage status = %d, body = %s", stage.Code, stage.Body.String()) + } + + list := httptest.NewRecorder() + handler.ServeHTTP(list, httptest.NewRequest(http.MethodGet, basePath, nil)) + if list.Code != http.StatusOK || strings.TrimSpace(list.Body.String()) != "[]" { + t.Fatalf("staged account became active: %d %s", list.Code, list.Body.String()) + } + + activate := request(basePath+"/migration/activate", `{ + "migrationId":"legacy-team", + "accountIds":["legacy-account"] + }`) + if activate.Code != http.StatusOK { + t.Fatalf("activate status = %d, body = %s", activate.Code, activate.Body.String()) + } + list = httptest.NewRecorder() + handler.ServeHTTP(list, httptest.NewRequest(http.MethodGet, basePath, nil)) + if list.Code != http.StatusOK || !strings.Contains(list.Body.String(), "legacy-account") { + t.Fatalf("activated account missing: %d %s", list.Code, list.Body.String()) + } + + rollback := request(basePath+"/migration/rollback", `{ + "migrationId":"legacy-team" + }`) + if rollback.Code != http.StatusOK { + t.Fatalf("rollback status = %d, body = %s", rollback.Code, rollback.Body.String()) + } + list = httptest.NewRecorder() + handler.ServeHTTP(list, httptest.NewRequest(http.MethodGet, basePath, nil)) + if list.Code != http.StatusOK || strings.TrimSpace(list.Body.String()) != "[]" { + t.Fatalf("rolled-back account remained active: %d %s", list.Code, list.Body.String()) + } +} + func TestStackLoginRejectsInvalidTokenAndMismatchedTeam(t *testing.T) { registry := tenant.NewRegistry(t.TempDir()) base := Server{MaxBodyBytes: 1024} diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go index fe5727db4..708c8ae14 100644 --- a/internal/proxy/proxy.go +++ b/internal/proxy/proxy.go @@ -1082,6 +1082,7 @@ func (s Server) handleAccounts(w http.ResponseWriter, r *http.Request) { ID string `json:"id"` Provider accounts.Provider `json:"provider"` AuthMode accounts.AuthMode `json:"auth_mode"` + Label string `json:"label,omitempty"` Email string `json:"email,omitempty"` Source string `json:"source"` } @@ -1092,6 +1093,7 @@ func (s Server) handleAccounts(w http.ResponseWriter, r *http.Request) { ID: account.ID, Provider: account.Provider, AuthMode: account.AuthMode, + Label: account.Label, Email: account.Email, Source: account.Source, })