From 0568f6b2b6933b2b7ba2d65d38c2ccf8e71bf7fc Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 3 Aug 2026 12:00:45 -0700 Subject: [PATCH 01/15] test: preserve legacy tenants during hosted cutover --- .../worker/test/tenant-migration.test.ts | 101 ++++++++++++++++++ internal/broker/hosted_client_test.go | 3 +- internal/proxy/multitenant_test.go | 53 +++++++++ 3 files changed, 156 insertions(+), 1 deletion(-) create mode 100644 cloudflare/packages/worker/test/tenant-migration.test.ts diff --git a/cloudflare/packages/worker/test/tenant-migration.test.ts b/cloudflare/packages/worker/test/tenant-migration.test.ts new file mode 100644 index 000000000..94cdcaa0f --- /dev/null +++ b/cloudflare/packages/worker/test/tenant-migration.test.ts @@ -0,0 +1,101 @@ +import { afterEach, describe, expect, test } from "bun:test" +import { + adminToken, + cleanupWorkers, + createTenant, + startWorker, +} from "./helpers/worker.ts" + +const servers: Array<{ stop: () => void }> = [] + +afterEach(async () => { + await cleanupWorkers() + for (const server of servers.splice(0)) server.stop() +}) + +describe("legacy tenant migration", () => { + test("moves credentials directly to an allowlisted hosted tenant without returning them", async () => { + const uploads: Array> = [] + const destination = Bun.serve({ + port: 0, + async fetch(request) { + expect(new URL(request.url).pathname).toBe("/_subrouter/accounts") + expect(request.headers.get("authorization")).toBe( + "Bearer srt_0123456789abcdef0123456789abcdef" + ) + uploads.push(await request.json() as Record) + return Response.json({ + account: { + id: uploads.at(-1)?.accountId, + kind: "codex", + label: uploads.at(-1)?.label, + }, + }) + }, + }) + servers.push(destination) + + const worker = await startWorker() + const tenant = await createTenant(worker.baseURL, "Legacy team") + const secrets = [ + "legacy-access-a", + "legacy-refresh-a", + "legacy-id-a", + "legacy-access-b", + "legacy-refresh-b", + "legacy-id-b", + ] + for (const suffix of ["a", "b"]) { + const response = await fetch(`${worker.baseURL}/tenant/accounts`, { + method: "POST", + headers: { + authorization: `Bearer ${tenant.key}`, + "content-type": "application/json", + }, + body: JSON.stringify({ + provider: "codex", + label: "Shared account", + tokens: { + accessToken: `legacy-access-${suffix}`, + refreshToken: `legacy-refresh-${suffix}`, + idToken: `legacy-id-${suffix}`, + accountID: `provider-${suffix}`, + }, + }), + }) + expect(response.status).toBe(200) + } + + const response = await fetch( + `${worker.baseURL}/admin/tenants/${tenant.id}/migrate-hosted`, + { + method: "POST", + headers: { + authorization: `Bearer ${adminToken}`, + "content-type": "application/json", + }, + body: JSON.stringify({ + destinationUrl: destination.url.origin, + tenantKey: "srt_0123456789abcdef0123456789abcdef", + }), + } + ) + expect(response.status).toBe(200) + const responseText = await response.text() + expect(JSON.parse(responseText)).toEqual({ ok: true, migrated: 2 }) + for (const secret of [...secrets, tenant.key]) { + expect(responseText).not.toContain(secret) + } + + expect(uploads).toHaveLength(2) + expect(new Set(uploads.map((upload) => upload.accountId)).size).toBe(2) + expect(uploads.map((upload) => upload.label)).toEqual([ + "Shared account", + "Shared account", + ]) + expect(uploads.map((upload) => upload.tokens?.refreshToken).sort()).toEqual([ + "legacy-refresh-a", + "legacy-refresh-b", + ]) + }, 60_000) +}) diff --git a/internal/broker/hosted_client_test.go b/internal/broker/hosted_client_test.go index cd648d562..5508acee2 100644 --- a/internal/broker/hosted_client_test.go +++ b/internal/broker/hosted_client_test.go @@ -24,6 +24,7 @@ func TestHostedClientUsesTenantScopedDirectAccountAPI(t *testing.T) { _ = json.NewEncoder(w).Encode([]map[string]any{{ "id": "user@example.com", "provider": "codex", "auth_mode": "oauth", "email": "user@example.com", + "label": "Shared Codex", "health": map[string]any{"ok": false, "message": "refresh failed"}, }}) case http.MethodPost: @@ -47,7 +48,7 @@ func TestHostedClientUsesTenantScopedDirectAccountAPI(t *testing.T) { if err != nil { t.Fatal(err) } - if len(items) != 1 || items[0].Kind != "codex" { + if len(items) != 1 || items[0].Kind != "codex" || items[0].Label != "Shared Codex" { t.Fatalf("items = %#v", items) } if items[0].Health == nil || items[0].Health.OK || diff --git a/internal/proxy/multitenant_test.go b/internal/proxy/multitenant_test.go index 0db25a74e..dff9827e1 100644 --- a/internal/proxy/multitenant_test.go +++ b/internal/proxy/multitenant_test.go @@ -12,6 +12,7 @@ import ( "net/url" "os" "path/filepath" + "sort" "strings" "sync/atomic" "testing" @@ -1274,6 +1275,58 @@ func TestTenantAccountUploadValidatesRepairTargetAndBodyShape(t *testing.T) { } } +func TestTenantAccountUploadPreservesDistinctMigrationIDsAndLabels(t *testing.T) { + registry, handler, _ := newMultiTenantFixture(t) + _, key, err := registry.Create("team") + if err != nil { + t.Fatal(err) + } + path := "/t/" + key + "/_subrouter/accounts" + for _, id := range []string{"legacy-account-a", "legacy-account-b"} { + body := fmt.Sprintf(`{ + "provider":"codex", + "accountId":%q, + "label":"Shared account", + "tokens":{ + "accessToken":"access-%s", + "refreshToken":"refresh-%s", + "idToken":"id-%s", + "accountID":"provider-%s" + } + }`, id, id, id, id, id) + response := httptest.NewRecorder() + handler.ServeHTTP( + response, + httptest.NewRequest(http.MethodPost, path, strings.NewReader(body)), + ) + if response.Code != http.StatusOK { + t.Fatalf("upload %s status = %d, body = %s", id, response.Code, response.Body.String()) + } + } + + response := httptest.NewRecorder() + handler.ServeHTTP(response, httptest.NewRequest(http.MethodGet, path, nil)) + if response.Code != http.StatusOK { + t.Fatalf("list status = %d, body = %s", response.Code, response.Body.String()) + } + var accounts []struct { + ID string `json:"id"` + Label string `json:"label"` + } + if err := json.Unmarshal(response.Body.Bytes(), &accounts); err != nil { + t.Fatal(err) + } + if len(accounts) != 2 { + t.Fatalf("accounts = %#v, want two distinct migrated accounts", accounts) + } + sort.Slice(accounts, func(i, j int) bool { return accounts[i].ID < accounts[j].ID }) + for i, id := range []string{"legacy-account-a", "legacy-account-b"} { + if accounts[i].ID != id || accounts[i].Label != "Shared account" { + t.Fatalf("account %d = %#v", i, accounts[i]) + } + } +} + func TestStackLoginRejectsInvalidTokenAndMismatchedTeam(t *testing.T) { registry := tenant.NewRegistry(t.TempDir()) base := Server{MaxBodyBytes: 1024} From 8f35319e6551a230a3847f99766b8ce1f2d23692 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 3 Aug 2026 12:04:50 -0700 Subject: [PATCH 02/15] fix: migrate legacy tenants without credential exposure --- cloudflare/packages/worker/src/index.ts | 59 +++++++ .../packages/worker/src/legacy-migration.ts | 164 ++++++++++++++++++ .../worker/test/tenant-migration.test.ts | 124 ++++++------- internal/accounts/codex_store.go | 7 +- internal/broker/client.go | 6 +- internal/broker/hosted_client_test.go | 2 +- internal/proxy/multitenant.go | 21 ++- internal/proxy/proxy.go | 2 + 8 files changed, 317 insertions(+), 68 deletions(-) create mode 100644 cloudflare/packages/worker/src/legacy-migration.ts diff --git a/cloudflare/packages/worker/src/index.ts b/cloudflare/packages/worker/src/index.ts index 9c6e690a9..55db53bcd 100644 --- a/cloudflare/packages/worker/src/index.ts +++ b/cloudflare/packages/worker/src/index.ts @@ -46,6 +46,10 @@ import { scheduleAxiomSpanExport, type MutableRequestTelemetryContext, } from "./telemetry.ts" +import { + migrateLegacyAccountsToHosted, + type LegacyMigrationAccount, +} from "./legacy-migration.ts" export { TenantRegistryDurableObject } @@ -2479,6 +2483,31 @@ export class SubrouterDurableObject extends DurableObject { } } + async listMigrationAccounts( + orgId: string + ): Promise<{ accounts: ReadonlyArray }> { + return { + accounts: this.ctx.storage.sql + .exec( + `SELECT * FROM accounts + WHERE org_id = ? + ORDER BY id`, + orgId + ) + .toArray() + .map((row) => ({ + id: row.id, + kind: row.kind as AccountKind, + label: row.label, + enabled: row.enabled === 1, + hasTotp: row.totp_seed_base32 !== null, + ...(row.credentials_json + ? { credentials: JSON.parse(row.credentials_json) as AccountCredentials } + : {}), + })), + } + } + async deleteAccount(input: { readonly orgId?: string; readonly accountId: string }): Promise<{ ok: true }> { const orgId = this.resolveOrgId(input.orgId) const accountId = this.requireNonEmpty(input.accountId, "accountId") @@ -4856,6 +4885,36 @@ const handleFetch = async ( return json(await registryActor(env).listTenants()) } + const tenantMigrationMatch = url.pathname.match( + /^\/admin\/tenants\/([^/]+)\/migrate-hosted$/ + ) + if (tenantMigrationMatch && request.method === "POST") { + const tenantId = decodeURIComponent(tenantMigrationMatch[1]!) + const record = await parseJsonRecord(request) + if (record instanceof Response) return record + try { + const { accounts } = await adminActor(env, tenantId) + .listMigrationAccounts(tenantId) + const requestHost = new URL(request.url).hostname + const migrated = await migrateLegacyAccountsToHosted({ + destinationUrl: record["destinationUrl"], + tenantKey: record["tenantKey"], + accounts, + allowLoopback: + requestHost === "localhost" || + requestHost === "127.0.0.1" || + requestHost === "[::1]", + }) + return json( + { ok: true, migrated }, + { headers: { "Cache-Control": "no-store" } } + ) + } catch (error) { + if (telemetry) telemetry.errorType = errorTypeFor(error) + return errorJson(error, 409) + } + } + const tenantRevokeMatch = url.pathname.match(/^\/admin\/tenants\/([^/]+)\/revoke$/) if (tenantRevokeMatch && request.method === "POST") { const tenantId = decodeURIComponent(tenantRevokeMatch[1]!) diff --git a/cloudflare/packages/worker/src/legacy-migration.ts b/cloudflare/packages/worker/src/legacy-migration.ts new file mode 100644 index 000000000..0a4ea887c --- /dev/null +++ b/cloudflare/packages/worker/src/legacy-migration.ts @@ -0,0 +1,164 @@ +import type { AccountCredentials, AccountKind } from "./contract.ts" + +export interface LegacyMigrationAccount { + readonly id: string + readonly kind: AccountKind + readonly label: string + readonly enabled: boolean + readonly hasTotp: boolean + readonly credentials?: AccountCredentials +} + +const productionOrigins = new Set([ + "https://sr.cmux.com", + "https://sr.cmux.dev", + "https://staging.sr.cmux.com", +]) +const tenantKeyPattern = /^srt_[0-9a-f]{32}$/ +const maxMigrationAccounts = 256 + +export async function migrateLegacyAccountsToHosted(options: { + readonly destinationUrl: unknown + readonly tenantKey: unknown + readonly accounts: ReadonlyArray + readonly allowLoopback?: boolean + readonly fetch?: typeof fetch +}): Promise { + const destination = migrationDestination( + options.destinationUrl, + options.allowLoopback ?? false + ) + if ( + typeof options.tenantKey !== "string" || + !tenantKeyPattern.test(options.tenantKey) + ) { + throw new Error("hosted migration tenant key is invalid") + } + if (options.accounts.length > maxMigrationAccounts) { + throw new Error("hosted migration account limit exceeded") + } + + // Validate every source row before the first external mutation. A retry can + // safely overwrite the same destination ids after a network interruption. + const uploads = options.accounts.map(migrationUpload) + const fetchImpl = options.fetch ?? fetch + for (const upload of uploads) { + let response: Response + try { + response = await fetchImpl(`${destination}/_subrouter/accounts`, { + method: "POST", + redirect: "error", + headers: { + authorization: `Bearer ${options.tenantKey}`, + "content-type": "application/json", + }, + body: JSON.stringify(upload), + }) + } catch { + throw new Error("hosted migration destination is unavailable") + } + if (!response.ok) { + throw new Error(`hosted migration upload failed (${response.status})`) + } + const body = await response.json().catch(() => null) + if ( + !body || + typeof body !== "object" || + !("account" in body) || + !body.account || + typeof body.account !== "object" || + !("id" in body.account) || + body.account.id !== upload.accountId + ) { + throw new Error("hosted migration returned an invalid account") + } + } + return uploads.length +} + +function migrationDestination(value: unknown, allowLoopback: boolean): string { + if (typeof value !== "string") { + throw new Error("hosted migration destination is invalid") + } + let url: URL + try { + url = new URL(value) + } catch { + throw new Error("hosted migration destination is invalid") + } + if ( + url.username || + url.password || + url.pathname !== "/" || + url.search || + url.hash + ) { + throw new Error("hosted migration destination is invalid") + } + const origin = url.origin + if (productionOrigins.has(origin)) return origin + if ( + allowLoopback && + url.protocol === "http:" && + (url.hostname === "localhost" || + url.hostname === "127.0.0.1" || + url.hostname === "[::1]") + ) { + return origin + } + throw new Error("hosted migration destination is not allowed") +} + +function migrationUpload(account: LegacyMigrationAccount): Record { + const id = normalizedString(account.id, 320) + const label = normalizedString(account.label, 320) + if (!id || !label || !account.enabled || account.hasTotp) { + throw new Error("legacy account cannot be migrated safely") + } + const credentials = account.credentials + if (!credentials) { + throw new Error("legacy account credentials are unavailable") + } + switch (account.kind) { + case "codex_oauth": { + const accessToken = requiredSecret(credentials.accessToken) + const refreshToken = requiredSecret(credentials.refreshToken) + const idToken = requiredSecret(credentials.idToken) + const accountID = requiredSecret(credentials.accountId) + return { + provider: "codex", + accountId: id, + label, + tokens: { accessToken, refreshToken, idToken, accountID }, + } + } + case "openai_apikey": + return { + provider: "openai-apikey", + accountId: id, + label, + apiKey: requiredSecret(credentials.apiKey), + } + case "anthropic_apikey": + return { + provider: "anthropic-apikey", + accountId: id, + label, + apiKey: requiredSecret(credentials.apiKey), + } + case "anthropic_oauth": + throw new Error("legacy Claude OAuth migration is not supported") + } +} + +function normalizedString(value: unknown, maxLength: number): string | null { + if (typeof value !== "string") return null + const normalized = value.trim() + return normalized && normalized.length <= maxLength ? normalized : null +} + +function requiredSecret(value: unknown): string { + const normalized = normalizedString(value, 64 * 1024) + if (!normalized) throw new Error("legacy account credentials are incomplete") + return normalized +} diff --git a/cloudflare/packages/worker/test/tenant-migration.test.ts b/cloudflare/packages/worker/test/tenant-migration.test.ts index 94cdcaa0f..2ec3e1b3b 100644 --- a/cloudflare/packages/worker/test/tenant-migration.test.ts +++ b/cloudflare/packages/worker/test/tenant-migration.test.ts @@ -1,4 +1,5 @@ import { afterEach, describe, expect, test } from "bun:test" +import { migrateLegacyAccountsToHosted } from "../src/legacy-migration.ts" import { adminToken, cleanupWorkers, @@ -6,65 +7,78 @@ import { startWorker, } from "./helpers/worker.ts" -const servers: Array<{ stop: () => void }> = [] - -afterEach(async () => { - await cleanupWorkers() - for (const server of servers.splice(0)) server.stop() -}) +afterEach(cleanupWorkers) describe("legacy tenant migration", () => { - test("moves credentials directly to an allowlisted hosted tenant without returning them", async () => { + test("moves credentials directly to an allowlisted hosted tenant", async () => { const uploads: Array> = [] - const destination = Bun.serve({ - port: 0, - async fetch(request) { - expect(new URL(request.url).pathname).toBe("/_subrouter/accounts") - expect(request.headers.get("authorization")).toBe( + const migrated = await migrateLegacyAccountsToHosted({ + destinationUrl: "https://sr.cmux.com", + tenantKey: "srt_0123456789abcdef0123456789abcdef", + accounts: ["a", "b"].map((suffix) => ({ + id: `legacy-account-${suffix}`, + kind: "codex_oauth" as const, + label: "Shared account", + enabled: true, + hasTotp: false, + credentials: { + accessToken: `legacy-access-${suffix}`, + refreshToken: `legacy-refresh-${suffix}`, + idToken: `legacy-id-${suffix}`, + accountId: `provider-${suffix}`, + }, + })), + fetch: (async (input, init) => { + expect(String(input)).toBe("https://sr.cmux.com/_subrouter/accounts") + expect(new Headers(init?.headers).get("authorization")).toBe( "Bearer srt_0123456789abcdef0123456789abcdef" ) - uploads.push(await request.json() as Record) + const upload = JSON.parse(String(init?.body)) as Record + uploads.push(upload) return Response.json({ account: { - id: uploads.at(-1)?.accountId, + id: upload.accountId, kind: "codex", - label: uploads.at(-1)?.label, + label: upload.label, }, }) - }, + }) as typeof fetch, }) - servers.push(destination) - const worker = await startWorker() - const tenant = await createTenant(worker.baseURL, "Legacy team") - const secrets = [ - "legacy-access-a", + expect(migrated).toBe(2) + expect(new Set(uploads.map((upload) => upload.accountId)).size).toBe(2) + expect(uploads.map((upload) => upload.label)).toEqual([ + "Shared account", + "Shared account", + ]) + expect(uploads.map((upload) => upload.tokens?.refreshToken).sort()).toEqual([ "legacy-refresh-a", - "legacy-id-a", - "legacy-access-b", "legacy-refresh-b", - "legacy-id-b", - ] - for (const suffix of ["a", "b"]) { - const response = await fetch(`${worker.baseURL}/tenant/accounts`, { - method: "POST", - headers: { - authorization: `Bearer ${tenant.key}`, - "content-type": "application/json", + ]) + }) + + test("admin route rejects untrusted destinations without exposing source credentials", async () => { + const worker = await startWorker() + const tenant = await createTenant(worker.baseURL, "Legacy team") + const secret = "legacy-refresh-secret" + const upload = await fetch(`${worker.baseURL}/tenant/accounts`, { + method: "POST", + headers: { + authorization: `Bearer ${tenant.key}`, + "content-type": "application/json", + }, + body: JSON.stringify({ + provider: "codex", + label: "Shared account", + tokens: { + accessToken: "legacy-access-secret", + refreshToken: secret, + idToken: "legacy-id-secret", + accountID: "provider-account", }, - body: JSON.stringify({ - provider: "codex", - label: "Shared account", - tokens: { - accessToken: `legacy-access-${suffix}`, - refreshToken: `legacy-refresh-${suffix}`, - idToken: `legacy-id-${suffix}`, - accountID: `provider-${suffix}`, - }, - }), - }) - expect(response.status).toBe(200) - } + }), + }) + expect(upload.status).toBe(200) const response = await fetch( `${worker.baseURL}/admin/tenants/${tenant.id}/migrate-hosted`, @@ -75,27 +89,15 @@ describe("legacy tenant migration", () => { "content-type": "application/json", }, body: JSON.stringify({ - destinationUrl: destination.url.origin, + destinationUrl: "https://attacker.example", tenantKey: "srt_0123456789abcdef0123456789abcdef", }), } ) - expect(response.status).toBe(200) + expect(response.status).toBe(409) const responseText = await response.text() - expect(JSON.parse(responseText)).toEqual({ ok: true, migrated: 2 }) - for (const secret of [...secrets, tenant.key]) { - expect(responseText).not.toContain(secret) - } - - expect(uploads).toHaveLength(2) - expect(new Set(uploads.map((upload) => upload.accountId)).size).toBe(2) - expect(uploads.map((upload) => upload.label)).toEqual([ - "Shared account", - "Shared account", - ]) - expect(uploads.map((upload) => upload.tokens?.refreshToken).sort()).toEqual([ - "legacy-refresh-a", - "legacy-refresh-b", - ]) + expect(responseText).toContain("destination is not allowed") + expect(responseText).not.toContain(secret) + expect(responseText).not.toContain(tenant.key) }, 60_000) }) diff --git a/internal/accounts/codex_store.go b/internal/accounts/codex_store.go index cac781124..52a145c4f 100644 --- a/internal/accounts/codex_store.go +++ b/internal/accounts/codex_store.go @@ -28,6 +28,7 @@ func (e *StorageKeyCollisionError) Error() string { type StoredCodexAccount struct { Email string `json:"email"` + Label string `json:"label,omitempty"` Provider Provider `json:"provider,omitempty"` AddedAt string `json:"addedAt"` Auth CodexAuthFile `json:"auth"` @@ -185,10 +186,14 @@ func (a StoredCodexAccount) toAccount(source string) (Account, bool) { } addedAt, _ := time.Parse(time.RFC3339, a.AddedAt) + label := strings.TrimSpace(a.Label) + if label == "" { + label = id + } out := Account{ ID: id, Provider: a.ProviderOrDefault(), - Label: id, + Label: label, Email: id, AddedAt: addedAt, Source: source, diff --git a/internal/broker/client.go b/internal/broker/client.go index 7d927da31..9ec1db9fd 100644 --- a/internal/broker/client.go +++ b/internal/broker/client.go @@ -199,6 +199,7 @@ func (c *Client) ListAccounts(ctx context.Context) ([]SharedAccount, error) { ID string `json:"id"` Provider string `json:"provider"` AuthMode string `json:"auth_mode"` + Label string `json:"label"` Email string `json:"email"` Health *struct { OK bool `json:"ok"` @@ -221,7 +222,10 @@ func (c *Client) ListAccounts(ctx context.Context) ([]SharedAccount, error) { return nil, fmt.Errorf("unsupported hosted account provider %q", item.Provider) } } - label := item.Email + label := item.Label + if label == "" { + label = item.Email + } if label == "" { label = item.ID } diff --git a/internal/broker/hosted_client_test.go b/internal/broker/hosted_client_test.go index 5508acee2..4937c8230 100644 --- a/internal/broker/hosted_client_test.go +++ b/internal/broker/hosted_client_test.go @@ -24,7 +24,7 @@ func TestHostedClientUsesTenantScopedDirectAccountAPI(t *testing.T) { _ = json.NewEncoder(w).Encode([]map[string]any{{ "id": "user@example.com", "provider": "codex", "auth_mode": "oauth", "email": "user@example.com", - "label": "Shared Codex", + "label": "Shared Codex", "health": map[string]any{"ok": false, "message": "refresh failed"}, }}) case http.MethodPost: diff --git a/internal/proxy/multitenant.go b/internal/proxy/multitenant.go index 1206c2d45..436a7ceff 100644 --- a/internal/proxy/multitenant.go +++ b/internal/proxy/multitenant.go @@ -674,6 +674,7 @@ func validStackTeamName(name string) bool { type tenantAccountUpload struct { Provider string `json:"provider"` + AccountID string `json:"accountId,omitempty"` Label string `json:"label"` APIKey string `json:"apiKey"` TargetAccountID string `json:"targetAccountID,omitempty"` @@ -707,12 +708,18 @@ func handleTenantAccountUpload(server *Server, w http.ResponseWriter, r *http.Re return } input.Provider = strings.ToLower(strings.TrimSpace(input.Provider)) + input.AccountID = strings.TrimSpace(input.AccountID) input.Label = strings.TrimSpace(input.Label) input.TargetAccountID = strings.TrimSpace(input.TargetAccountID) if input.Label == "" || len(input.Label) > 320 { http.Error(w, "account label is required", http.StatusBadRequest) return } + if input.AccountID != "" && + (len(input.AccountID) > 320 || containsTerminalControl(input.AccountID)) { + http.Error(w, "account id is invalid", http.StatusBadRequest) + return + } var id string var kind string validateRepairTarget := func(candidate string) bool { @@ -726,13 +733,16 @@ func handleTenantAccountUpload(server *Server, w http.ResponseWriter, r *http.Re http.Error(w, "complete Codex OAuth tokens are required", http.StatusBadRequest) return } - id, kind = input.Label, "codex" + id, kind = input.AccountID, "codex" + if id == "" { + id = input.Label + } if !validateRepairTarget(id) { http.Error(w, "repair target does not match uploaded account", http.StatusConflict) return } err := server.AccountRef.store.SaveStored(accounts.StoredCodexAccount{ - Email: input.Label, Provider: accounts.ProviderCodex, + Email: id, Label: input.Label, Provider: accounts.ProviderCodex, Auth: accounts.CodexAuthFile{ AuthMode: "chatgpt", Tokens: &accounts.CodexTokens{ @@ -754,13 +764,16 @@ func handleTenantAccountUpload(server *Server, w http.ResponseWriter, r *http.Re if input.Provider == "anthropic-apikey" { provider = accounts.ProviderClaude } - id, kind = "apikey:"+input.Provider+":"+input.Label, input.Provider + id, kind = input.AccountID, input.Provider + if id == "" { + id = "apikey:" + input.Provider + ":" + input.Label + } if !validateRepairTarget(id) { http.Error(w, "repair target does not match uploaded account", http.StatusConflict) return } if err := server.AccountRef.store.SaveStored(accounts.StoredCodexAccount{ - Email: id, Provider: provider, + Email: id, Label: input.Label, Provider: provider, Auth: accounts.CodexAuthFile{AuthMode: "apikey", OpenAIAPIKey: strings.TrimSpace(input.APIKey)}, }); err != nil { http.Error(w, "save API key", http.StatusInternalServerError) diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go index fe5727db4..708c8ae14 100644 --- a/internal/proxy/proxy.go +++ b/internal/proxy/proxy.go @@ -1082,6 +1082,7 @@ func (s Server) handleAccounts(w http.ResponseWriter, r *http.Request) { ID string `json:"id"` Provider accounts.Provider `json:"provider"` AuthMode accounts.AuthMode `json:"auth_mode"` + Label string `json:"label,omitempty"` Email string `json:"email,omitempty"` Source string `json:"source"` } @@ -1092,6 +1093,7 @@ func (s Server) handleAccounts(w http.ResponseWriter, r *http.Request) { ID: account.ID, Provider: account.Provider, AuthMode: account.AuthMode, + Label: account.Label, Email: account.Email, Source: account.Source, }) From 52ba3f6d18539eef49000f71cb3c2d4f50bb04f1 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 3 Aug 2026 12:08:26 -0700 Subject: [PATCH 03/15] test: finalize legacy credential migration safely --- .../worker/test/tenant-migration.test.ts | 78 ++++++++++++++++++- 1 file changed, 77 insertions(+), 1 deletion(-) diff --git a/cloudflare/packages/worker/test/tenant-migration.test.ts b/cloudflare/packages/worker/test/tenant-migration.test.ts index 2ec3e1b3b..c98497c5b 100644 --- a/cloudflare/packages/worker/test/tenant-migration.test.ts +++ b/cloudflare/packages/worker/test/tenant-migration.test.ts @@ -1,5 +1,8 @@ import { afterEach, describe, expect, test } from "bun:test" -import { migrateLegacyAccountsToHosted } from "../src/legacy-migration.ts" +import { + migrateLegacyAccountsToHosted, + migrateLegacyTenant, +} from "../src/legacy-migration.ts" import { adminToken, cleanupWorkers, @@ -57,6 +60,65 @@ describe("legacy tenant migration", () => { ]) }) + test("quiesces and removes source credentials only after every upload succeeds", async () => { + let sourceState: "active" | "quiesced" | "removed" = "active" + const accounts = [legacyAccount("a"), legacyAccount("b")] + const migrated = await migrateLegacyTenant({ + destinationUrl: "https://sr.cmux.com", + tenantKey: "srt_0123456789abcdef0123456789abcdef", + finalizeSource: true, + source: { + list: async () => accounts, + begin: async () => { + sourceState = "quiesced" + return accounts + }, + complete: async () => { + expect(sourceState).toBe("quiesced") + sourceState = "removed" + }, + restore: async () => { + sourceState = "active" + }, + }, + fetch: (async (_input, init) => { + const upload = JSON.parse(String(init?.body)) as Record + return Response.json({ account: { id: upload.accountId } }) + }) as typeof fetch, + }) + + expect(migrated).toBe(2) + expect(sourceState).toBe("removed") + }) + + test("restores source routing when a finalized migration upload fails", async () => { + let sourceState: "active" | "quiesced" | "removed" = "active" + await expect( + migrateLegacyTenant({ + destinationUrl: "https://sr.cmux.com", + tenantKey: "srt_0123456789abcdef0123456789abcdef", + finalizeSource: true, + source: { + list: async () => [legacyAccount("a")], + begin: async () => { + sourceState = "quiesced" + return [legacyAccount("a")] + }, + complete: async () => { + sourceState = "removed" + }, + restore: async () => { + sourceState = "active" + }, + }, + fetch: (async () => { + throw new Error("network body with secret") + }) as typeof fetch, + }) + ).rejects.toThrow("destination is unavailable") + expect(sourceState).toBe("active") + }) + test("admin route rejects untrusted destinations without exposing source credentials", async () => { const worker = await startWorker() const tenant = await createTenant(worker.baseURL, "Legacy team") @@ -101,3 +163,17 @@ describe("legacy tenant migration", () => { expect(responseText).not.toContain(tenant.key) }, 60_000) }) + +const legacyAccount = (suffix: string) => ({ + id: `legacy-account-${suffix}`, + kind: "codex_oauth" as const, + label: "Shared account", + enabled: true, + hasTotp: false, + credentials: { + accessToken: `legacy-access-${suffix}`, + refreshToken: `legacy-refresh-${suffix}`, + idToken: `legacy-id-${suffix}`, + accountId: `provider-${suffix}`, + }, +}) From 8d2bf9263eb8dfe386e73de2a973208711492ed6 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 3 Aug 2026 12:17:44 -0700 Subject: [PATCH 04/15] fix: finalize legacy credential migration safely --- cloudflare/packages/worker/src/index.ts | 226 +++++++++++++++--- .../packages/worker/src/legacy-migration.ts | 50 +++- .../worker/test/tenant-migration.test.ts | 116 +++++++-- 3 files changed, 339 insertions(+), 53 deletions(-) diff --git a/cloudflare/packages/worker/src/index.ts b/cloudflare/packages/worker/src/index.ts index 55db53bcd..a8aea9856 100644 --- a/cloudflare/packages/worker/src/index.ts +++ b/cloudflare/packages/worker/src/index.ts @@ -47,7 +47,7 @@ import { type MutableRequestTelemetryContext, } from "./telemetry.ts" import { - migrateLegacyAccountsToHosted, + migrateLegacyTenant, type LegacyMigrationAccount, } from "./legacy-migration.ts" @@ -115,6 +115,23 @@ interface CredentialLeaseEventInput { readonly retryAt?: number } +interface LegacyMigrationActor { + listMigrationAccounts( + orgId: string + ): Promise<{ accounts: ReadonlyArray }> + beginMigrationAccounts( + orgId: string + ): Promise<{ accounts: ReadonlyArray }> + completeMigrationAccounts(input: { + readonly orgId: string + readonly accounts: ReadonlyArray + }): Promise + restoreMigrationAccounts(input: { + readonly orgId: string + readonly accounts: ReadonlyArray + }): Promise +} + interface CredentialLeaseRow { readonly [key: string]: SqlStorageValue readonly id: string @@ -440,6 +457,48 @@ const parseJsonRecord = async ( return body as Record } +const parseBoundedJsonRecord = async ( + request: Request, + maxBytes: number +): Promise | Response> => { + const contentLength = Number(request.headers.get("content-length") ?? "0") + if (Number.isFinite(contentLength) && contentLength > maxBytes) { + return json({ error: "Request body too large" }, { status: 413 }) + } + if (!request.body) { + return json({ error: "Missing JSON body" }, { status: 400 }) + } + const reader = request.body.getReader() + const chunks: Uint8Array[] = [] + let total = 0 + for (;;) { + const { done, value } = await reader.read() + if (done) break + total += value.byteLength + if (total > maxBytes) { + await reader.cancel().catch(() => {}) + return json({ error: "Request body too large" }, { status: 413 }) + } + chunks.push(value) + } + const body = new Uint8Array(total) + let offset = 0 + for (const chunk of chunks) { + body.set(chunk, offset) + offset += chunk.byteLength + } + let decoded: unknown + try { + decoded = JSON.parse(new TextDecoder().decode(body)) + } catch { + return json({ error: "Missing JSON body" }, { status: 400 }) + } + if (!decoded || typeof decoded !== "object" || Array.isArray(decoded)) { + return json({ error: "Missing JSON body" }, { status: 400 }) + } + return decoded as Record +} + const parseRouteInput = async (request: Request): Promise => { const body = await request.json().catch(() => ({})) if (!body || typeof body !== "object" || Array.isArray(body)) { @@ -2508,40 +2567,72 @@ export class SubrouterDurableObject extends DurableObject { } } + async beginMigrationAccounts( + orgId: string + ): Promise<{ accounts: ReadonlyArray }> { + return await this.ctx.blockConcurrencyWhile(async () => { + // A refresh token can be single-use. Let every refresh already holding + // one settle before snapshotting, then prevent later refreshes by + // disabling the source rows in the same concurrency barrier. + await Promise.allSettled([...this.refreshInFlight.values()]) + const snapshot = await this.listMigrationAccounts(orgId) + this.ctx.storage.sql.exec( + "UPDATE accounts SET enabled = 0, updated_at = ? WHERE org_id = ?", + Date.now(), + orgId + ) + await this.scheduleNextRefreshAlarm() + return snapshot + }) + } + + async restoreMigrationAccounts(input: { + readonly orgId: string + readonly accounts: ReadonlyArray + }): Promise { + this.ctx.storage.transactionSync(() => { + for (const account of input.accounts) { + this.ctx.storage.sql.exec( + "UPDATE accounts SET enabled = ?, updated_at = ? WHERE id = ? AND org_id = ?", + account.enabled ? 1 : 0, + Date.now(), + account.id, + input.orgId + ) + } + }) + await this.scheduleNextRefreshAlarm() + } + + async completeMigrationAccounts(input: { + readonly orgId: string + readonly accounts: ReadonlyArray + }): Promise { + this.ctx.storage.transactionSync(() => { + const sql = this.ctx.storage.sql + for (const account of input.accounts) { + const row = this.getAccountRow(sql, input.orgId, account.id, false) + if ( + !row || + row.enabled !== 0 || + row.credentials_json !== JSON.stringify(account.credentials) + ) { + throw new Error("migration source changed during hosted upload") + } + } + for (const account of input.accounts) { + this.deleteAccountRows(sql, input.orgId, account.id) + } + }) + await this.scheduleNextRefreshAlarm() + } + async deleteAccount(input: { readonly orgId?: string; readonly accountId: string }): Promise<{ ok: true }> { const orgId = this.resolveOrgId(input.orgId) const accountId = this.requireNonEmpty(input.accountId, "accountId") const row = this.getAccountRow(this.ctx.storage.sql, orgId, accountId, false) if (!row) throw new Error("account not found") - const sql = this.ctx.storage.sql - sql.exec("DELETE FROM accounts WHERE id = ? AND org_id = ?", accountId, orgId) - sql.exec("DELETE FROM account_model_quotas WHERE account_id = ?", accountId) - sql.exec("DELETE FROM account_usage WHERE account_id = ?", accountId) - sql.exec( - "DELETE FROM session_assignments WHERE org_id = ? AND account_id = ?", - orgId, - accountId - ) - sql.exec( - "DELETE FROM sticky_sessions WHERE org_id = ? AND account_id = ?", - orgId, - accountId - ) - sql.exec( - "DELETE FROM sticky_session_routes WHERE org_id = ? AND account_id = ?", - orgId, - accountId - ) - sql.exec( - "DELETE FROM credential_leases WHERE org_id = ? AND account_id = ?", - orgId, - accountId - ) - sql.exec( - "DELETE FROM credential_quota_cooldowns WHERE org_id = ? AND account_id = ?", - orgId, - accountId - ) + this.deleteAccountRows(this.ctx.storage.sql, orgId, accountId) await this.scheduleNextRefreshAlarm() return { ok: true } } @@ -3162,6 +3253,41 @@ export class SubrouterDurableObject extends DurableObject { ) } + private deleteAccountRows( + sql: SqlStorage, + orgId: string, + accountId: string + ): void { + sql.exec("DELETE FROM accounts WHERE id = ? AND org_id = ?", accountId, orgId) + sql.exec("DELETE FROM account_model_quotas WHERE account_id = ?", accountId) + sql.exec("DELETE FROM account_usage WHERE account_id = ?", accountId) + sql.exec( + "DELETE FROM session_assignments WHERE org_id = ? AND account_id = ?", + orgId, + accountId + ) + sql.exec( + "DELETE FROM sticky_sessions WHERE org_id = ? AND account_id = ?", + orgId, + accountId + ) + sql.exec( + "DELETE FROM sticky_session_routes WHERE org_id = ? AND account_id = ?", + orgId, + accountId + ) + sql.exec( + "DELETE FROM credential_leases WHERE org_id = ? AND account_id = ?", + orgId, + accountId + ) + sql.exec( + "DELETE FROM credential_quota_cooldowns WHERE org_id = ? AND account_id = ?", + orgId, + accountId + ) + } + private async refreshAccountIfExpired( orgId: string, accountId: string, @@ -4890,23 +5016,51 @@ const handleFetch = async ( ) if (tenantMigrationMatch && request.method === "POST") { const tenantId = decodeURIComponent(tenantMigrationMatch[1]!) - const record = await parseJsonRecord(request) + const record = await parseBoundedJsonRecord(request, 4 * 1024) if (record instanceof Response) return record + if ( + record["finalizeSource"] !== undefined && + typeof record["finalizeSource"] !== "boolean" + ) { + return json({ error: "finalizeSource must be boolean" }, { status: 400 }) + } try { - const { accounts } = await adminActor(env, tenantId) - .listMigrationAccounts(tenantId) + // Cloudflare's generated RPC stub erases methods that carry the + // encrypted credential union. Keep the cast at this trusted DO + // boundary and validate the rows before any outbound upload. + const actor = adminActor( + env, + tenantId + ) as unknown as LegacyMigrationActor const requestHost = new URL(request.url).hostname - const migrated = await migrateLegacyAccountsToHosted({ + const finalizeSource = record["finalizeSource"] === true + const migrated = await migrateLegacyTenant({ destinationUrl: record["destinationUrl"], tenantKey: record["tenantKey"], - accounts, + finalizeSource, allowLoopback: requestHost === "localhost" || requestHost === "127.0.0.1" || requestHost === "[::1]", + source: { + list: async () => + (await actor.listMigrationAccounts(tenantId)).accounts, + begin: async () => + (await actor.beginMigrationAccounts(tenantId)).accounts, + complete: async (accounts) => + await actor.completeMigrationAccounts({ + orgId: tenantId, + accounts, + }), + restore: async (accounts) => + await actor.restoreMigrationAccounts({ + orgId: tenantId, + accounts, + }), + }, }) return json( - { ok: true, migrated }, + { ok: true, migrated, sourceFinalized: finalizeSource }, { headers: { "Cache-Control": "no-store" } } ) } catch (error) { diff --git a/cloudflare/packages/worker/src/legacy-migration.ts b/cloudflare/packages/worker/src/legacy-migration.ts index 0a4ea887c..b42ea0ff2 100644 --- a/cloudflare/packages/worker/src/legacy-migration.ts +++ b/cloudflare/packages/worker/src/legacy-migration.ts @@ -9,14 +9,59 @@ export interface LegacyMigrationAccount { readonly credentials?: AccountCredentials } +export interface LegacyMigrationSource { + readonly list: () => Promise> + readonly begin: () => Promise> + readonly complete: ( + accounts: ReadonlyArray + ) => Promise + readonly restore: ( + accounts: ReadonlyArray + ) => Promise +} + const productionOrigins = new Set([ "https://sr.cmux.com", "https://sr.cmux.dev", - "https://staging.sr.cmux.com", ]) const tenantKeyPattern = /^srt_[0-9a-f]{32}$/ const maxMigrationAccounts = 256 +export async function migrateLegacyTenant(options: { + readonly destinationUrl: unknown + readonly tenantKey: unknown + readonly finalizeSource: boolean + readonly allowLoopback?: boolean + readonly source: LegacyMigrationSource + readonly fetch?: typeof fetch +}): Promise { + const accounts = options.finalizeSource + ? await options.source.begin() + : await options.source.list() + try { + const migrated = await migrateLegacyAccountsToHosted({ + destinationUrl: options.destinationUrl, + tenantKey: options.tenantKey, + accounts, + allowLoopback: options.allowLoopback, + fetch: options.fetch, + }) + if (options.finalizeSource) { + await options.source.complete(accounts) + } + return migrated + } catch (error) { + if (options.finalizeSource) { + try { + await options.source.restore(accounts) + } catch { + throw new Error("hosted migration failed and source restoration failed") + } + } + throw error + } +} + export async function migrateLegacyAccountsToHosted(options: { readonly destinationUrl: unknown readonly tenantKey: unknown @@ -47,7 +92,8 @@ export async function migrateLegacyAccountsToHosted(options: { try { response = await fetchImpl(`${destination}/_subrouter/accounts`, { method: "POST", - redirect: "error", + redirect: "manual", + signal: AbortSignal.timeout(10_000), headers: { authorization: `Bearer ${options.tenantKey}`, "content-type": "application/json", diff --git a/cloudflare/packages/worker/test/tenant-migration.test.ts b/cloudflare/packages/worker/test/tenant-migration.test.ts index c98497c5b..3b627a986 100644 --- a/cloudflare/packages/worker/test/tenant-migration.test.ts +++ b/cloudflare/packages/worker/test/tenant-migration.test.ts @@ -10,7 +10,12 @@ import { startWorker, } from "./helpers/worker.ts" -afterEach(cleanupWorkers) +const servers: Array<{ stop: () => void }> = [] + +afterEach(async () => { + await cleanupWorkers() + for (const server of servers.splice(0)) server.stop() +}) describe("legacy tenant migration", () => { test("moves credentials directly to an allowlisted hosted tenant", async () => { @@ -61,7 +66,9 @@ describe("legacy tenant migration", () => { }) test("quiesces and removes source credentials only after every upload succeeds", async () => { - let sourceState: "active" | "quiesced" | "removed" = "active" + const sourceState = { + value: "active" as "active" | "quiesced" | "removed", + } const accounts = [legacyAccount("a"), legacyAccount("b")] const migrated = await migrateLegacyTenant({ destinationUrl: "https://sr.cmux.com", @@ -70,15 +77,15 @@ describe("legacy tenant migration", () => { source: { list: async () => accounts, begin: async () => { - sourceState = "quiesced" + sourceState.value = "quiesced" return accounts }, complete: async () => { - expect(sourceState).toBe("quiesced") - sourceState = "removed" + expect(sourceState.value).toBe("quiesced") + sourceState.value = "removed" }, restore: async () => { - sourceState = "active" + sourceState.value = "active" }, }, fetch: (async (_input, init) => { @@ -88,11 +95,13 @@ describe("legacy tenant migration", () => { }) expect(migrated).toBe(2) - expect(sourceState).toBe("removed") + expect(sourceState.value).toBe("removed") }) test("restores source routing when a finalized migration upload fails", async () => { - let sourceState: "active" | "quiesced" | "removed" = "active" + const sourceState = { + value: "active" as "active" | "quiesced" | "removed", + } await expect( migrateLegacyTenant({ destinationUrl: "https://sr.cmux.com", @@ -101,24 +110,101 @@ describe("legacy tenant migration", () => { source: { list: async () => [legacyAccount("a")], begin: async () => { - sourceState = "quiesced" + sourceState.value = "quiesced" return [legacyAccount("a")] }, complete: async () => { - sourceState = "removed" + sourceState.value = "removed" }, restore: async () => { - sourceState = "active" + sourceState.value = "active" }, }, - fetch: (async () => { - throw new Error("network body with secret") - }) as typeof fetch, + fetch: Object.assign( + async () => { + throw new Error("network body with secret") + }, + { preconnect() {} } + ) as typeof fetch, }) ).rejects.toThrow("destination is unavailable") - expect(sourceState).toBe("active") + expect(sourceState.value).toBe("active") }) + test("finalizes source rows through the admin route after hosted upload", async () => { + const uploads: Array> = [] + const destination = Bun.serve({ + port: 0, + async fetch(request) { + const upload = (await request.json()) as Record + uploads.push(upload) + return Response.json({ account: { id: upload.accountId } }) + }, + }) + servers.push(destination) + const worker = await startWorker() + const tenant = await createTenant(worker.baseURL, "Finalized team") + const secret = "legacy-finalization-refresh-secret" + const upload = await fetch(`${worker.baseURL}/tenant/accounts`, { + method: "POST", + headers: { + authorization: `Bearer ${tenant.key}`, + "content-type": "application/json", + }, + body: JSON.stringify({ + provider: "codex", + label: "Shared account", + tokens: { + accessToken: "legacy-finalization-access", + refreshToken: secret, + idToken: "legacy-finalization-id", + accountID: "legacy-provider-account", + }, + }), + }) + expect(upload.status).toBe(200) + + const response = await fetch( + `${worker.baseURL}/admin/tenants/${tenant.id}/migrate-hosted`, + { + method: "POST", + headers: { + authorization: `Bearer ${adminToken}`, + "content-type": "application/json", + }, + body: JSON.stringify({ + destinationUrl: destination.url.origin, + tenantKey: "srt_0123456789abcdef0123456789abcdef", + finalizeSource: true, + }), + } + ) + const responseText = await response.text() + expect({ status: response.status, body: responseText }).toEqual({ + status: 200, + body: JSON.stringify({ + ok: true, + migrated: 1, + sourceFinalized: true, + }), + }) + expect(JSON.parse(responseText)).toEqual({ + ok: true, + migrated: 1, + sourceFinalized: true, + }) + expect(responseText).not.toContain(secret) + expect(uploads).toHaveLength(1) + expect(uploads[0]?.label).toBe("Shared account") + + const source = await fetch(`${worker.baseURL}/tenant/accounts`, { + headers: { authorization: `Bearer ${tenant.key}` }, + }) + expect(source.status).toBe(200) + const sourceBody = (await source.json()) as { accounts: unknown[] } + expect(sourceBody).toEqual({ accounts: [] }) + }, 60_000) + test("admin route rejects untrusted destinations without exposing source credentials", async () => { const worker = await startWorker() const tenant = await createTenant(worker.baseURL, "Legacy team") From ffa88c03706c8c0d133307c4c6eccc67fcd3bd06 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 3 Aug 2026 12:29:56 -0700 Subject: [PATCH 05/15] test: cover interrupted hosted migration finalization --- .../worker/test/tenant-migration.test.ts | 200 ++++++++++++++++++ 1 file changed, 200 insertions(+) diff --git a/cloudflare/packages/worker/test/tenant-migration.test.ts b/cloudflare/packages/worker/test/tenant-migration.test.ts index 3b627a986..2014fdb51 100644 --- a/cloudflare/packages/worker/test/tenant-migration.test.ts +++ b/cloudflare/packages/worker/test/tenant-migration.test.ts @@ -8,6 +8,8 @@ import { cleanupWorkers, createTenant, startWorker, + stopWorker, + waitForCondition, } from "./helpers/worker.ts" const servers: Array<{ stop: () => void }> = [] @@ -131,6 +133,31 @@ describe("legacy tenant migration", () => { expect(sourceState.value).toBe("active") }) + test("restores source routing when quiescing fails after changing state", async () => { + const sourceState = { + value: "active" as "active" | "quiesced", + } + await expect( + migrateLegacyTenant({ + destinationUrl: "https://sr.cmux.com", + tenantKey: "srt_0123456789abcdef0123456789abcdef", + finalizeSource: true, + source: { + list: async () => [legacyAccount("a")], + begin: async () => { + sourceState.value = "quiesced" + throw new Error("alarm update failed") + }, + complete: async () => {}, + restore: async () => { + sourceState.value = "active" + }, + }, + }) + ).rejects.toThrow("alarm update failed") + expect(sourceState.value).toBe("active") + }) + test("finalizes source rows through the admin route after hosted upload", async () => { const uploads: Array> = [] const destination = Bun.serve({ @@ -205,6 +232,112 @@ describe("legacy tenant migration", () => { expect(sourceBody).toEqual({ accounts: [] }) }, 60_000) + test("serializes a concurrent account mutation after source finalization", async () => { + let destinationStarted = false + let releaseDestination!: () => void + const destinationReleased = new Promise((resolve) => { + releaseDestination = resolve + }) + const destination = Bun.serve({ + port: 0, + async fetch(request) { + destinationStarted = true + await destinationReleased + const upload = (await request.json()) as Record + return Response.json({ account: { id: upload.accountId } }) + }, + }) + servers.push(destination) + const worker = await startWorker() + const tenant = await createTenant(worker.baseURL, "Concurrent team") + const created = await uploadLegacyCodexAccount( + worker.baseURL, + tenant.key, + "Original account", + "original" + ) + + const migration = migrateTenant(worker.baseURL, tenant.id, { + destinationUrl: destination.url.origin, + finalizeSource: true, + }) + await waitForCondition( + () => destinationStarted, + "hosted migration upload to start" + ) + const mutation = fetch(`${worker.baseURL}/admin/accounts`, { + method: "POST", + headers: { + authorization: `Bearer ${adminToken}`, + "content-type": "application/json", + }, + body: JSON.stringify({ + id: created.id, + orgId: tenant.id, + kind: "codex_oauth", + label: "Concurrent account", + enabled: true, + credentials: codexCredentials("concurrent"), + }), + }) + await Bun.sleep(50) + releaseDestination() + + const [migrationResponse, mutationResponse] = await Promise.all([ + migration, + mutation, + ]) + expect(migrationResponse.status).toBe(200) + expect(mutationResponse.status).toBe(200) + const accounts = await listAdminAccounts( + worker.baseURL, + tenant.id + ) + expect(accounts.map((account) => account.label)).toEqual([ + "Concurrent account", + ]) + }, 60_000) + + test("restores a quiesced source after the Worker restarts mid-upload", async () => { + let destinationStarted = false + const destination = Bun.serve({ + port: 0, + fetch() { + destinationStarted = true + return new Promise(() => {}) + }, + }) + servers.push(destination) + const worker = await startWorker() + const tenant = await createTenant(worker.baseURL, "Restarted team") + await uploadLegacyCodexAccount( + worker.baseURL, + tenant.key, + "Restarted account", + "restart" + ) + + const migration = migrateTenant(worker.baseURL, tenant.id, { + destinationUrl: destination.url.origin, + finalizeSource: true, + }).catch(() => null) + await waitForCondition( + () => destinationStarted, + "hosted migration upload to start" + ) + await stopWorker(worker) + await migration + + const restarted = await startWorker({ persistDir: worker.persistDir }) + const accounts = await listAdminAccounts(restarted.baseURL, tenant.id) + expect(accounts).toHaveLength(1) + expect(accounts[0]).toMatchObject({ + label: "Restarted account", + enabled: true, + hasCredentials: true, + }) + }, 60_000) + test("admin route rejects untrusted destinations without exposing source credentials", async () => { const worker = await startWorker() const tenant = await createTenant(worker.baseURL, "Legacy team") @@ -263,3 +396,70 @@ const legacyAccount = (suffix: string) => ({ accountId: `provider-${suffix}`, }, }) + +const codexCredentials = (suffix: string) => ({ + accessToken: `legacy-access-${suffix}`, + refreshToken: `legacy-refresh-${suffix}`, + idToken: `legacy-id-${suffix}`, + accountId: `provider-${suffix}`, +}) + +const uploadLegacyCodexAccount = async ( + baseURL: string, + tenantKey: string, + label: string, + suffix: string +): Promise<{ id: string }> => { + const response = await fetch(`${baseURL}/tenant/accounts`, { + method: "POST", + headers: { + authorization: `Bearer ${tenantKey}`, + "content-type": "application/json", + }, + body: JSON.stringify({ + provider: "codex", + label, + tokens: { + ...codexCredentials(suffix), + accountID: `provider-${suffix}`, + }, + }), + }) + expect(response.status).toBe(200) + return (await response.json()) as { id: string } +} + +const migrateTenant = async ( + baseURL: string, + tenantId: string, + input: { + readonly destinationUrl: string + readonly finalizeSource: boolean + } +): Promise => + await fetch(`${baseURL}/admin/tenants/${tenantId}/migrate-hosted`, { + method: "POST", + headers: { + authorization: `Bearer ${adminToken}`, + "content-type": "application/json", + }, + body: JSON.stringify({ + ...input, + tenantKey: "srt_0123456789abcdef0123456789abcdef", + }), + }) + +const listAdminAccounts = async ( + baseURL: string, + tenantId: string +): Promise>> => { + const response = await fetch( + `${baseURL}/admin/accounts?tenant=${encodeURIComponent(tenantId)}`, + { headers: { authorization: `Bearer ${adminToken}` } } + ) + expect(response.status).toBe(200) + const body = (await response.json()) as { + accounts: Array> + } + return body.accounts +} From d6693fdb00324e745184c5e201bd7d7247faad5a Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 3 Aug 2026 12:34:52 -0700 Subject: [PATCH 06/15] fix: make hosted migration failure atomic --- cloudflare/packages/worker/src/index.ts | 301 ++++++++++++------ .../packages/worker/src/legacy-migration.ts | 52 ++- 2 files changed, 238 insertions(+), 115 deletions(-) diff --git a/cloudflare/packages/worker/src/index.ts b/cloudflare/packages/worker/src/index.ts index a8aea9856..c30f82055 100644 --- a/cloudflare/packages/worker/src/index.ts +++ b/cloudflare/packages/worker/src/index.ts @@ -115,23 +115,6 @@ interface CredentialLeaseEventInput { readonly retryAt?: number } -interface LegacyMigrationActor { - listMigrationAccounts( - orgId: string - ): Promise<{ accounts: ReadonlyArray }> - beginMigrationAccounts( - orgId: string - ): Promise<{ accounts: ReadonlyArray }> - completeMigrationAccounts(input: { - readonly orgId: string - readonly accounts: ReadonlyArray - }): Promise - restoreMigrationAccounts(input: { - readonly orgId: string - readonly accounts: ReadonlyArray - }): Promise -} - interface CredentialLeaseRow { readonly [key: string]: SqlStorageValue readonly id: string @@ -294,6 +277,13 @@ interface AccountRow { readonly consecutive_quota_errors: number | null } +interface HostedMigrationStateRow { + readonly [key: string]: SqlStorageValue + readonly org_id: string + readonly accounts_json: string + readonly started_at: number +} + interface TableInfoRow { readonly [key: string]: SqlStorageValue readonly name: string @@ -1698,6 +1688,7 @@ const generateTotp = async ( export class SubrouterDurableObject extends DurableObject { private readonly refreshInFlight = new Map>() + private migrationPreparing = false constructor(ctx: DurableObjectState, env: Env) { super(ctx, env) @@ -1816,12 +1807,18 @@ export class SubrouterDurableObject extends DurableObject { draining INTEGER NOT NULL DEFAULT 0, started_at INTEGER NOT NULL ); + CREATE TABLE IF NOT EXISTS hosted_migration_state( + org_id TEXT PRIMARY KEY, + accounts_json TEXT NOT NULL, + started_at INTEGER NOT NULL + ); INSERT OR IGNORE INTO subrouter_status(id) VALUES (1); INSERT OR IGNORE INTO lifecycle(id, started_at) VALUES (1, ${Date.now()}); `) this.ensureAccountHealthColumns() this.ctx.blockConcurrencyWhile(async () => { + this.restoreInterruptedHostedMigrations() this.pruneCredentialLeases(Date.now()) await this.scheduleNextRefreshAlarm() }) @@ -2542,89 +2539,156 @@ export class SubrouterDurableObject extends DurableObject { } } - async listMigrationAccounts( - orgId: string - ): Promise<{ accounts: ReadonlyArray }> { - return { - accounts: this.ctx.storage.sql - .exec( - `SELECT * FROM accounts - WHERE org_id = ? - ORDER BY id`, - orgId - ) - .toArray() - .map((row) => ({ - id: row.id, - kind: row.kind as AccountKind, - label: row.label, - enabled: row.enabled === 1, - hasTotp: row.totp_seed_base32 !== null, - ...(row.credentials_json - ? { credentials: JSON.parse(row.credentials_json) as AccountCredentials } - : {}), - })), + async migrateHosted(input: { + readonly orgId: string + readonly destinationUrl: unknown + readonly tenantKey: unknown + readonly finalizeSource: boolean + readonly allowLoopback: boolean + }): Promise<{ migrated: number }> { + const orgId = this.resolveOrgId(input.orgId) + const migrate = async (): Promise<{ migrated: number }> => ({ + migrated: await migrateLegacyTenant({ + destinationUrl: input.destinationUrl, + tenantKey: input.tenantKey, + finalizeSource: input.finalizeSource, + allowLoopback: input.allowLoopback, + source: { + list: async () => this.listMigrationAccounts(orgId), + begin: async () => this.beginMigrationAccounts(orgId), + complete: async (accounts) => + this.completeMigrationAccounts(orgId, accounts), + restore: async () => this.restoreMigrationAccounts(orgId), + }, + }), + }) + if (!input.finalizeSource) return await migrate() + if (this.migrationPreparing) { + throw new Error("hosted migration is already in progress") + } + + // Stop new refreshes before waiting for the ones that already own a + // single-use refresh token. No source row changes until they settle. + this.migrationPreparing = true + try { + await Promise.allSettled([...this.refreshInFlight.values()]) + // Cloudflare bounds this barrier at 30 seconds. Migration uploads run in + // parallel with a 10-second request deadline, leaving room for the two + // local transactions while excluding every concurrent tenant mutation. + return await this.ctx.blockConcurrencyWhile(migrate) + } finally { + this.migrationPreparing = false } } - async beginMigrationAccounts( + private listMigrationAccounts( orgId: string - ): Promise<{ accounts: ReadonlyArray }> { - return await this.ctx.blockConcurrencyWhile(async () => { - // A refresh token can be single-use. Let every refresh already holding - // one settle before snapshotting, then prevent later refreshes by - // disabling the source rows in the same concurrency barrier. - await Promise.allSettled([...this.refreshInFlight.values()]) - const snapshot = await this.listMigrationAccounts(orgId) + ): ReadonlyArray { + return this.ctx.storage.sql + .exec( + `SELECT * FROM accounts + WHERE org_id = ? + ORDER BY id`, + orgId + ) + .toArray() + .map((row) => ({ + id: row.id, + kind: row.kind as AccountKind, + label: row.label, + enabled: row.enabled === 1, + hasTotp: row.totp_seed_base32 !== null, + updatedAt: row.updated_at, + ...(row.credentials_json + ? { credentials: JSON.parse(row.credentials_json) as AccountCredentials } + : {}), + })) + } + + private beginMigrationAccounts( + orgId: string + ): ReadonlyArray { + const accounts = this.listMigrationAccounts(orgId) + const accountsJSON = JSON.stringify(accounts) + this.ctx.storage.transactionSync(() => { + const existing = this.ctx.storage.sql + .exec( + "SELECT COUNT(*) AS count FROM hosted_migration_state WHERE org_id = ?", + orgId + ) + .one().count + if (existing !== 0) { + throw new Error("hosted migration recovery is pending") + } this.ctx.storage.sql.exec( - "UPDATE accounts SET enabled = 0, updated_at = ? WHERE org_id = ?", - Date.now(), + `INSERT INTO hosted_migration_state(org_id, accounts_json, started_at) + VALUES (?, ?, ?)`, + orgId, + accountsJSON, + Date.now() + ) + this.ctx.storage.sql.exec( + "UPDATE accounts SET enabled = 0 WHERE org_id = ?", orgId ) - await this.scheduleNextRefreshAlarm() - return snapshot }) + return accounts } - async restoreMigrationAccounts(input: { - readonly orgId: string - readonly accounts: ReadonlyArray - }): Promise { + private restoreMigrationAccounts(orgId: string): void { this.ctx.storage.transactionSync(() => { - for (const account of input.accounts) { + const state = this.hostedMigrationState(orgId) + if (!state) return + const accounts = JSON.parse( + state.accounts_json + ) as ReadonlyArray + for (const account of accounts) { this.ctx.storage.sql.exec( - "UPDATE accounts SET enabled = ?, updated_at = ? WHERE id = ? AND org_id = ?", + "UPDATE accounts SET enabled = ? WHERE id = ? AND org_id = ?", account.enabled ? 1 : 0, - Date.now(), account.id, - input.orgId + orgId ) } + this.ctx.storage.sql.exec( + "DELETE FROM hosted_migration_state WHERE org_id = ?", + orgId + ) }) - await this.scheduleNextRefreshAlarm() } - async completeMigrationAccounts(input: { - readonly orgId: string - readonly accounts: ReadonlyArray - }): Promise { + private completeMigrationAccounts( + orgId: string, + accounts: ReadonlyArray + ): void { this.ctx.storage.transactionSync(() => { const sql = this.ctx.storage.sql - for (const account of input.accounts) { - const row = this.getAccountRow(sql, input.orgId, account.id, false) - if ( - !row || - row.enabled !== 0 || - row.credentials_json !== JSON.stringify(account.credentials) - ) { - throw new Error("migration source changed during hosted upload") - } + const state = this.hostedMigrationState(orgId) + if (!state || state.accounts_json !== JSON.stringify(accounts)) { + throw new Error("hosted migration source snapshot is unavailable") } - for (const account of input.accounts) { - this.deleteAccountRows(sql, input.orgId, account.id) + const rows = sql + .exec( + "SELECT * FROM accounts WHERE org_id = ? ORDER BY id", + orgId + ) + .toArray() + if ( + rows.length !== accounts.length || + rows.some((row, index) => + !this.migrationAccountMatchesRow(accounts[index], row) + ) + ) { + throw new Error("migration source changed during hosted upload") + } + for (const account of accounts) { + this.deleteAccountRows(sql, orgId, account.id) } + sql.exec( + "DELETE FROM hosted_migration_state WHERE org_id = ?", + orgId + ) }) - await this.scheduleNextRefreshAlarm() } async deleteAccount(input: { readonly orgId?: string; readonly accountId: string }): Promise<{ ok: true }> { @@ -3253,6 +3317,64 @@ export class SubrouterDurableObject extends DurableObject { ) } + private hostedMigrationState( + orgId: string + ): HostedMigrationStateRow | null { + return this.ctx.storage.sql + .exec( + "SELECT * FROM hosted_migration_state WHERE org_id = ?", + orgId + ) + .toArray()[0] ?? null + } + + private migrationAccountMatchesRow( + account: LegacyMigrationAccount | undefined, + row: AccountRow + ): boolean { + if (!account) return false + const credentialsJSON = account.credentials === undefined + ? null + : JSON.stringify(account.credentials) + return ( + row.id === account.id && + row.kind === account.kind && + row.label === account.label && + row.enabled === 0 && + (row.totp_seed_base32 !== null) === account.hasTotp && + row.credentials_json === credentialsJSON && + row.updated_at === account.updatedAt + ) + } + + private restoreInterruptedHostedMigrations(): void { + const states = this.ctx.storage.sql + .exec( + "SELECT * FROM hosted_migration_state ORDER BY org_id" + ) + .toArray() + if (states.length === 0) return + this.ctx.storage.transactionSync(() => { + for (const state of states) { + const accounts = JSON.parse( + state.accounts_json + ) as ReadonlyArray + for (const account of accounts) { + this.ctx.storage.sql.exec( + "UPDATE accounts SET enabled = ? WHERE id = ? AND org_id = ?", + account.enabled ? 1 : 0, + account.id, + state.org_id + ) + } + this.ctx.storage.sql.exec( + "DELETE FROM hosted_migration_state WHERE org_id = ?", + state.org_id + ) + } + }) + } + private deleteAccountRows( sql: SqlStorage, orgId: string, @@ -3304,6 +3426,7 @@ export class SubrouterDurableObject extends DurableObject { const key = `${orgId}:${accountId}` const existing = this.refreshInFlight.get(key) if (existing) return existing + if (this.migrationPreparing) return row const promise = this.refreshAccountRow(row, force).finally(() => { this.refreshInFlight.delete(key) }) @@ -5025,16 +5148,10 @@ const handleFetch = async ( return json({ error: "finalizeSource must be boolean" }, { status: 400 }) } try { - // Cloudflare's generated RPC stub erases methods that carry the - // encrypted credential union. Keep the cast at this trusted DO - // boundary and validate the rows before any outbound upload. - const actor = adminActor( - env, - tenantId - ) as unknown as LegacyMigrationActor const requestHost = new URL(request.url).hostname const finalizeSource = record["finalizeSource"] === true - const migrated = await migrateLegacyTenant({ + const { migrated } = await adminActor(env, tenantId).migrateHosted({ + orgId: tenantId, destinationUrl: record["destinationUrl"], tenantKey: record["tenantKey"], finalizeSource, @@ -5042,22 +5159,6 @@ const handleFetch = async ( requestHost === "localhost" || requestHost === "127.0.0.1" || requestHost === "[::1]", - source: { - list: async () => - (await actor.listMigrationAccounts(tenantId)).accounts, - begin: async () => - (await actor.beginMigrationAccounts(tenantId)).accounts, - complete: async (accounts) => - await actor.completeMigrationAccounts({ - orgId: tenantId, - accounts, - }), - restore: async (accounts) => - await actor.restoreMigrationAccounts({ - orgId: tenantId, - accounts, - }), - }, }) return json( { ok: true, migrated, sourceFinalized: finalizeSource }, diff --git a/cloudflare/packages/worker/src/legacy-migration.ts b/cloudflare/packages/worker/src/legacy-migration.ts index b42ea0ff2..75bc3cae5 100644 --- a/cloudflare/packages/worker/src/legacy-migration.ts +++ b/cloudflare/packages/worker/src/legacy-migration.ts @@ -6,6 +6,7 @@ export interface LegacyMigrationAccount { readonly label: string readonly enabled: boolean readonly hasTotp: boolean + readonly updatedAt?: number readonly credentials?: AccountCredentials } @@ -25,7 +26,7 @@ const productionOrigins = new Set([ "https://sr.cmux.dev", ]) const tenantKeyPattern = /^srt_[0-9a-f]{32}$/ -const maxMigrationAccounts = 256 +const maxMigrationAccounts = 16 export async function migrateLegacyTenant(options: { readonly destinationUrl: unknown @@ -35,18 +36,31 @@ export async function migrateLegacyTenant(options: { readonly source: LegacyMigrationSource readonly fetch?: typeof fetch }): Promise { - const accounts = options.finalizeSource - ? await options.source.begin() - : await options.source.list() + // Reject operator input before quiescing any source credential. + const destinationUrl = migrationDestination( + options.destinationUrl, + options.allowLoopback ?? false + ) + const tenantKey = migrationTenantKey(options.tenantKey) + let accounts: ReadonlyArray = [] + let sourceBegan = false try { + if (options.finalizeSource) { + // Mark the attempt before calling begin so a source with persisted + // recovery state can undo a failure that happens after quiescing. + sourceBegan = true + accounts = await options.source.begin() + } else { + accounts = await options.source.list() + } const migrated = await migrateLegacyAccountsToHosted({ - destinationUrl: options.destinationUrl, - tenantKey: options.tenantKey, + destinationUrl, + tenantKey, accounts, allowLoopback: options.allowLoopback, fetch: options.fetch, }) - if (options.finalizeSource) { + if (sourceBegan) { await options.source.complete(accounts) } return migrated @@ -73,12 +87,7 @@ export async function migrateLegacyAccountsToHosted(options: { options.destinationUrl, options.allowLoopback ?? false ) - if ( - typeof options.tenantKey !== "string" || - !tenantKeyPattern.test(options.tenantKey) - ) { - throw new Error("hosted migration tenant key is invalid") - } + const tenantKey = migrationTenantKey(options.tenantKey) if (options.accounts.length > maxMigrationAccounts) { throw new Error("hosted migration account limit exceeded") } @@ -87,7 +96,7 @@ export async function migrateLegacyAccountsToHosted(options: { // safely overwrite the same destination ids after a network interruption. const uploads = options.accounts.map(migrationUpload) const fetchImpl = options.fetch ?? fetch - for (const upload of uploads) { + const results = await Promise.allSettled(uploads.map(async (upload) => { let response: Response try { response = await fetchImpl(`${destination}/_subrouter/accounts`, { @@ -95,7 +104,7 @@ export async function migrateLegacyAccountsToHosted(options: { redirect: "manual", signal: AbortSignal.timeout(10_000), headers: { - authorization: `Bearer ${options.tenantKey}`, + authorization: `Bearer ${tenantKey}`, "content-type": "application/json", }, body: JSON.stringify(upload), @@ -118,10 +127,23 @@ export async function migrateLegacyAccountsToHosted(options: { ) { throw new Error("hosted migration returned an invalid account") } + })) + const failed = results.find( + (result): result is PromiseRejectedResult => result.status === "rejected" + ) + if (failed) { + throw failed.reason } return uploads.length } +function migrationTenantKey(value: unknown): string { + if (typeof value !== "string" || !tenantKeyPattern.test(value)) { + throw new Error("hosted migration tenant key is invalid") + } + return value +} + function migrationDestination(value: unknown, allowLoopback: boolean): string { if (typeof value !== "string") { throw new Error("hosted migration destination is invalid") From 74743f4b8055c4523c7b4655635384a068d89226 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 3 Aug 2026 12:47:54 -0700 Subject: [PATCH 07/15] test: keep migrated OAuth credentials staged --- .../worker/test/tenant-migration.test.ts | 481 ++++++++++++++++-- internal/accounts/codex_store_test.go | 170 +++++++ internal/proxy/multitenant_test.go | 131 +++++ 3 files changed, 747 insertions(+), 35 deletions(-) diff --git a/cloudflare/packages/worker/test/tenant-migration.test.ts b/cloudflare/packages/worker/test/tenant-migration.test.ts index 2014fdb51..ff562b635 100644 --- a/cloudflare/packages/worker/test/tenant-migration.test.ts +++ b/cloudflare/packages/worker/test/tenant-migration.test.ts @@ -20,8 +20,9 @@ afterEach(async () => { }) describe("legacy tenant migration", () => { - test("moves credentials directly to an allowlisted hosted tenant", async () => { + test("stages credentials without activating them before source finalization", async () => { const uploads: Array> = [] + const calls: string[] = [] const migrated = await migrateLegacyAccountsToHosted({ destinationUrl: "https://sr.cmux.com", tenantKey: "srt_0123456789abcdef0123456789abcdef", @@ -39,18 +40,20 @@ describe("legacy tenant migration", () => { }, })), fetch: (async (input, init) => { - expect(String(input)).toBe("https://sr.cmux.com/_subrouter/accounts") + expect(String(input)).toBe( + "https://sr.cmux.com/_subrouter/accounts/migration/stage" + ) expect(new Headers(init?.headers).get("authorization")).toBe( "Bearer srt_0123456789abcdef0123456789abcdef" ) - const upload = JSON.parse(String(init?.body)) as Record - uploads.push(upload) + calls.push(new URL(String(input)).pathname) + const body = JSON.parse(String(init?.body)) as Record + uploads.push(...body.accounts) return Response.json({ - account: { - id: upload.accountId, - kind: "codex", - label: upload.label, - }, + ok: true, + accountIds: body.accounts.map( + (upload: Record) => upload.accountId + ), }) }) as typeof fetch, }) @@ -65,6 +68,7 @@ describe("legacy tenant migration", () => { "legacy-refresh-a", "legacy-refresh-b", ]) + expect(calls).toEqual(["/_subrouter/accounts/migration/stage"]) }) test("quiesces and removes source credentials only after every upload succeeds", async () => { @@ -90,10 +94,7 @@ describe("legacy tenant migration", () => { sourceState.value = "active" }, }, - fetch: (async (_input, init) => { - const upload = JSON.parse(String(init?.body)) as Record - return Response.json({ account: { id: upload.accountId } }) - }) as typeof fetch, + fetch: hostedMigrationFetch(), }) expect(migrated).toBe(2) @@ -122,15 +123,125 @@ describe("legacy tenant migration", () => { sourceState.value = "active" }, }, - fetch: Object.assign( - async () => { - throw new Error("network body with secret") + fetch: hostedMigrationFetch({ failStage: true }), + }) + ).rejects.toThrow("destination is unavailable") + expect(sourceState.value).toBe("active") + }) + + test("restores source routing when staging and inactive rollback both fail", async () => { + const sourceState = { + value: "active" as "active" | "quiesced" | "removed", + } + let recoveryPreserved = false + await expect( + migrateLegacyTenant({ + destinationUrl: "https://sr.cmux.com", + tenantKey: "srt_0123456789abcdef0123456789abcdef", + finalizeSource: true, + source: { + list: async () => [legacyAccount("a")], + begin: async () => { + sourceState.value = "quiesced" + return [legacyAccount("a")] + }, + complete: async () => { + sourceState.value = "removed" + }, + restore: async (_accounts, options) => { + sourceState.value = "active" + recoveryPreserved = options.preserveRecovery }, - { preconnect() {} } - ) as typeof fetch, + }, + fetch: hostedMigrationFetch({ + failStage: true, + failRollback: true, + }), }) ).rejects.toThrow("destination is unavailable") expect(sourceState.value).toBe("active") + expect(recoveryPreserved).toBe(true) + }) + + test("clears recovery after an authoritatively rejected stage", async () => { + const sourceState = { + value: "active" as "active" | "quiesced", + } + const destinationCalls: string[] = [] + let recoveryPreserved = true + await expect( + migrateLegacyTenant({ + destinationUrl: "https://sr.cmux.com", + tenantKey: "srt_0123456789abcdef0123456789abcdef", + finalizeSource: true, + source: { + list: async () => [legacyAccount("a")], + begin: async () => { + sourceState.value = "quiesced" + return [legacyAccount("a")] + }, + complete: async () => {}, + restore: async (_accounts, options) => { + sourceState.value = "active" + recoveryPreserved = options.preserveRecovery + }, + }, + fetch: hostedMigrationFetch({ + rejectAuthorization: true, + calls: destinationCalls, + }), + }) + ).rejects.toThrow("stage failed (401)") + expect(sourceState.value).toBe("active") + expect(recoveryPreserved).toBe(false) + expect(destinationCalls).toEqual([ + "/_subrouter/accounts/migration/stage", + ]) + }) + + test("rejects unsafe migration identities before quiescing the source", async () => { + const invalidAccounts = [ + { ...legacyAccount("control"), label: "unsafe\u001b[31m" }, + { ...legacyAccount("label"), label: "é".repeat(161) }, + { ...legacyAccount("id"), id: "é".repeat(161) }, + ] + for (const account of invalidAccounts) { + let sourceBegan = false + const destinationCalls: string[] = [] + await expect( + migrateLegacyTenant({ + destinationUrl: "https://sr.cmux.com", + tenantKey: "srt_0123456789abcdef0123456789abcdef", + finalizeSource: true, + source: { + list: async () => [account], + begin: async () => { + sourceBegan = true + return [account] + }, + complete: async () => {}, + restore: async () => {}, + }, + fetch: hostedMigrationFetch({ calls: destinationCalls }), + }) + ).rejects.toThrow("legacy account cannot be migrated safely") + expect(sourceBegan).toBe(false) + expect(destinationCalls).toEqual([]) + } + }) + + test("accepts migration identities at the 320-byte UTF-8 boundary", async () => { + const migrated = await migrateLegacyAccountsToHosted({ + destinationUrl: "https://sr.cmux.com", + tenantKey: "srt_0123456789abcdef0123456789abcdef", + accounts: [{ + ...legacyAccount("boundary"), + id: "é".repeat(160), + label: "é".repeat(160), + }], + fetch: hostedMigrationFetch(), + }) + expect(migrated).toBe(1) }) test("restores source routing when quiescing fails after changing state", async () => { @@ -160,12 +271,12 @@ describe("legacy tenant migration", () => { test("finalizes source rows through the admin route after hosted upload", async () => { const uploads: Array> = [] + const destinationCalls: string[] = [] const destination = Bun.serve({ port: 0, async fetch(request) { - const upload = (await request.json()) as Record - uploads.push(upload) - return Response.json({ account: { id: upload.accountId } }) + destinationCalls.push(new URL(request.url).pathname) + return await hostedMigrationResponse(request, { uploads }) }, }) servers.push(destination) @@ -223,6 +334,10 @@ describe("legacy tenant migration", () => { expect(responseText).not.toContain(secret) expect(uploads).toHaveLength(1) expect(uploads[0]?.label).toBe("Shared account") + expect(destinationCalls).toEqual([ + "/_subrouter/accounts/migration/stage", + "/_subrouter/accounts/migration/activate", + ]) const source = await fetch(`${worker.baseURL}/tenant/accounts`, { headers: { authorization: `Bearer ${tenant.key}` }, @@ -232,7 +347,107 @@ describe("legacy tenant migration", () => { expect(sourceBody).toEqual({ accounts: [] }) }, 60_000) - test("serializes a concurrent account mutation after source finalization", async () => { + test("rejects finalization while preflight staging is in progress", async () => { + let firstStageStarted = false + let releaseFirstStage!: () => void + const firstStageReleased = new Promise((resolve) => { + releaseFirstStage = resolve + }) + let stageCount = 0 + const destination = Bun.serve({ + port: 0, + async fetch(request) { + if (new URL(request.url).pathname.endsWith("/stage")) { + stageCount += 1 + if (stageCount === 1) { + firstStageStarted = true + await firstStageReleased + } + } + return await hostedMigrationResponse(request) + }, + }) + servers.push(destination) + const worker = await startWorker() + const tenant = await createTenant(worker.baseURL, "Serialized team") + await uploadLegacyCodexAccount( + worker.baseURL, + tenant.key, + "Serialized account", + "serialized" + ) + + const preflight = migrateTenant(worker.baseURL, tenant.id, { + destinationUrl: destination.url.origin, + finalizeSource: false, + }) + await waitForCondition( + () => firstStageStarted, + "hosted migration preflight to start" + ) + const concurrentFinalization = await migrateTenant( + worker.baseURL, + tenant.id, + { + destinationUrl: destination.url.origin, + finalizeSource: true, + } + ) + releaseFirstStage() + const preflightResponse = await preflight + + expect(concurrentFinalization.status).toBe(409) + expect(preflightResponse.status).toBe(200) + expect(stageCount).toBe(1) + + const finalization = await migrateTenant(worker.baseURL, tenant.id, { + destinationUrl: destination.url.origin, + finalizeSource: true, + }) + expect(finalization.status).toBe(200) + }, 60_000) + + test("returns a completed receipt without touching Hosted on retry", async () => { + const destinationCalls: string[] = [] + const destination = Bun.serve({ + port: 0, + async fetch(request) { + destinationCalls.push(new URL(request.url).pathname) + return await hostedMigrationResponse(request) + }, + }) + servers.push(destination) + const worker = await startWorker() + const tenant = await createTenant(worker.baseURL, "Completed team") + await uploadLegacyCodexAccount( + worker.baseURL, + tenant.key, + "Completed account", + "completed" + ) + + const first = await migrateTenant(worker.baseURL, tenant.id, { + destinationUrl: destination.url.origin, + finalizeSource: true, + }) + expect(first.status).toBe(200) + const callsAfterSuccess = [...destinationCalls] + + const retry = await migrateTenant(worker.baseURL, tenant.id, { + destinationUrl: destination.url.origin, + finalizeSource: true, + }) + expect(retry.status).toBe(200) + const retryBody = (await retry.json()) as Record + expect(retryBody).toEqual({ + ok: true, + migrated: 1, + sourceFinalized: true, + }) + expect(destinationCalls).toEqual(callsAfterSuccess) + }, 60_000) + + test("rejects a concurrent account mutation after source finalization", async () => { let destinationStarted = false let releaseDestination!: () => void const destinationReleased = new Promise((resolve) => { @@ -241,10 +456,11 @@ describe("legacy tenant migration", () => { const destination = Bun.serve({ port: 0, async fetch(request) { - destinationStarted = true - await destinationReleased - const upload = (await request.json()) as Record - return Response.json({ account: { id: upload.accountId } }) + if (new URL(request.url).pathname.endsWith("/stage")) { + destinationStarted = true + await destinationReleased + } + return await hostedMigrationResponse(request) }, }) servers.push(destination) @@ -280,7 +496,6 @@ describe("legacy tenant migration", () => { credentials: codexCredentials("concurrent"), }), }) - await Bun.sleep(50) releaseDestination() const [migrationResponse, mutationResponse] = await Promise.all([ @@ -288,23 +503,28 @@ describe("legacy tenant migration", () => { mutation, ]) expect(migrationResponse.status).toBe(200) - expect(mutationResponse.status).toBe(200) + expect(mutationResponse.status).toBe(400) const accounts = await listAdminAccounts( worker.baseURL, tenant.id ) - expect(accounts.map((account) => account.label)).toEqual([ - "Concurrent account", - ]) + expect(accounts).toEqual([]) }, 60_000) - test("restores a quiesced source after the Worker restarts mid-upload", async () => { + test("retains recovery binding after restart until matching rollback", async () => { let destinationStarted = false + let hangStage = true + const destinationCalls: string[] = [] const destination = Bun.serve({ port: 0, - fetch() { - destinationStarted = true - return new Promise(() => {}) + async fetch(request) { + const path = new URL(request.url).pathname + destinationCalls.push(path) + if (path.endsWith("/stage") && hangStage) { + destinationStarted = true + return new Promise(() => {}) + } + return await hostedMigrationResponse(request) }, }) servers.push(destination) @@ -336,6 +556,142 @@ describe("legacy tenant migration", () => { enabled: true, hasCredentials: true, }) + + const wrongDestinationCalls: string[] = [] + const wrongDestination = Bun.serve({ + port: 0, + async fetch(request) { + wrongDestinationCalls.push(new URL(request.url).pathname) + return await hostedMigrationResponse(request) + }, + }) + servers.push(wrongDestination) + const wrongRetry = await migrateTenant(restarted.baseURL, tenant.id, { + destinationUrl: wrongDestination.url.origin, + finalizeSource: true, + }) + expect(wrongRetry.status).toBe(409) + expect(wrongDestinationCalls).toEqual([]) + + hangStage = false + const callsBeforeRetry = destinationCalls.length + const retry = await migrateTenant(restarted.baseURL, tenant.id, { + destinationUrl: destination.url.origin, + finalizeSource: true, + }) + expect(retry.status).toBe(200) + expect(destinationCalls.slice(callsBeforeRetry)).toEqual([ + "/_subrouter/accounts/migration/rollback", + "/_subrouter/accounts/migration/stage", + "/_subrouter/accounts/migration/activate", + ]) + }, 60_000) + + test("keeps an ambiguous activation quiesced until rollback is confirmed", async () => { + let activationStarted = false + let hangActivation = true + const destinationCalls: string[] = [] + const destination = Bun.serve({ + port: 0, + async fetch(request) { + const path = new URL(request.url).pathname + destinationCalls.push(path) + if (path.endsWith("/activate") && hangActivation) { + activationStarted = true + return new Promise(() => {}) + } + return await hostedMigrationResponse(request) + }, + }) + servers.push(destination) + const worker = await startWorker() + const tenant = await createTenant(worker.baseURL, "Ambiguous team") + await uploadLegacyCodexAccount( + worker.baseURL, + tenant.key, + "Ambiguous account", + "ambiguous" + ) + + const migration = migrateTenant(worker.baseURL, tenant.id, { + destinationUrl: destination.url.origin, + finalizeSource: true, + }).catch(() => null) + await waitForCondition( + () => activationStarted, + "hosted migration activation to start" + ) + await stopWorker(worker) + await migration + + const restarted = await startWorker({ persistDir: worker.persistDir }) + const quiesced = await listAdminAccounts(restarted.baseURL, tenant.id) + expect(quiesced).toHaveLength(1) + expect(quiesced[0]?.enabled).toBe(false) + + hangActivation = false + const retry = await migrateTenant(restarted.baseURL, tenant.id, { + destinationUrl: destination.url.origin, + finalizeSource: true, + }) + expect(retry.status).toBe(200) + expect(destinationCalls).toContain( + "/_subrouter/accounts/migration/rollback" + ) + expect(await listAdminAccounts(restarted.baseURL, tenant.id)).toEqual([]) + }, 60_000) + + test("rejects recovery against a different Hosted destination", async () => { + let activationStarted = false + const originalDestination = Bun.serve({ + port: 0, + async fetch(request) { + if (new URL(request.url).pathname.endsWith("/activate")) { + activationStarted = true + return new Promise(() => {}) + } + return await hostedMigrationResponse(request) + }, + }) + const wrongDestinationCalls: string[] = [] + const wrongDestination = Bun.serve({ + port: 0, + async fetch(request) { + wrongDestinationCalls.push(new URL(request.url).pathname) + return await hostedMigrationResponse(request) + }, + }) + servers.push(originalDestination, wrongDestination) + const worker = await startWorker() + const tenant = await createTenant(worker.baseURL, "Bound team") + await uploadLegacyCodexAccount( + worker.baseURL, + tenant.key, + "Bound account", + "bound" + ) + + const migration = migrateTenant(worker.baseURL, tenant.id, { + destinationUrl: originalDestination.url.origin, + finalizeSource: true, + }).catch(() => null) + await waitForCondition( + () => activationStarted, + "bound migration activation to start" + ) + await stopWorker(worker) + await migration + + const restarted = await startWorker({ persistDir: worker.persistDir }) + const retry = await migrateTenant(restarted.baseURL, tenant.id, { + destinationUrl: wrongDestination.url.origin, + finalizeSource: true, + }) + expect(retry.status).toBe(409) + expect(wrongDestinationCalls).toEqual([]) + const accounts = await listAdminAccounts(restarted.baseURL, tenant.id) + expect(accounts).toHaveLength(1) + expect(accounts[0]?.enabled).toBe(false) }, 60_000) test("admin route rejects untrusted destinations without exposing source credentials", async () => { @@ -463,3 +819,58 @@ const listAdminAccounts = async ( } return body.accounts } + +const hostedMigrationFetch = ( + options: { + readonly failStage?: boolean + readonly failRollback?: boolean + readonly rejectAuthorization?: boolean + readonly calls?: string[] + } = {} +): typeof fetch => + Object.assign( + async (input: Parameters[0], init?: Parameters[1]) => { + const request = new Request(String(input), init) + options.calls?.push(new URL(request.url).pathname) + if (options.rejectAuthorization) { + return new Response("unauthorized", { status: 401 }) + } + if ( + options.failStage && + new URL(request.url).pathname.endsWith("/stage") + ) { + throw new Error("network body with secret") + } + if ( + options.failRollback && + new URL(request.url).pathname.endsWith("/rollback") + ) { + throw new Error("rollback is unavailable") + } + return await hostedMigrationResponse(request) + }, + { preconnect() {} } + ) as typeof fetch + +const hostedMigrationResponse = async ( + request: Request, + options: { readonly uploads?: Array> } = {} +): Promise => { + const path = new URL(request.url).pathname + const body = (await request.json()) as Record + if (path.endsWith("/stage")) { + const accounts = body.accounts as Array> + options.uploads?.push(...accounts) + return Response.json({ + ok: true, + accountIds: accounts.map((account) => account.accountId), + }) + } + if (path.endsWith("/activate")) { + return Response.json({ ok: true, activated: body.accountIds }) + } + if (path.endsWith("/rollback")) { + return Response.json({ ok: true, rolledBack: true }) + } + return new Response("not found", { status: 404 }) +} diff --git a/internal/accounts/codex_store_test.go b/internal/accounts/codex_store_test.go index fe75057c5..f49fc2dee 100644 --- a/internal/accounts/codex_store_test.go +++ b/internal/accounts/codex_store_test.go @@ -1,10 +1,180 @@ package accounts import ( + "fmt" "os" "testing" + "time" ) +func TestCodexStoreMigrationBatchVisibilityIsAtomicForConcurrentReaders(t *testing.T) { + store := CodexStore{Dir: t.TempDir()} + const accountCount = 64 + staged := make([]StoredCodexAccount, 0, accountCount) + for i := 0; i < accountCount; i++ { + id := fmt.Sprintf("migration-%03d@example.com", i) + staged = append(staged, StoredCodexAccount{ + Email: id, + Provider: ProviderCodex, + Auth: CodexAuthFile{ + AuthMode: "apikey", + OpenAIAPIKey: "sk-test", + }, + }) + } + const batchID = "atomic-visibility" + if err := store.StageMigrationBatch(batchID, staged); err != nil { + t.Fatal(err) + } + + stop := make(chan struct{}) + done := make(chan struct{}) + errors := make(chan error, 1) + go func() { + defer close(done) + marker := store.migrationBatchMarker(batchID) + body := []byte(`{"accountIds":[]}` + "\n") + for { + select { + case <-stop: + return + default: + } + if err := writeFileAtomic(marker, body, 0o600); err != nil { + errors <- err + return + } + if err := os.Remove(marker); err != nil { + errors <- err + return + } + } + }() + defer func() { + close(stop) + <-done + }() + + for i := 0; i < 500; i++ { + select { + case err := <-errors: + t.Fatal(err) + default: + } + accounts, err := store.ListStored() + if err != nil { + t.Fatal(err) + } + if len(accounts) != 0 && len(accounts) != accountCount { + t.Fatalf("concurrent batch list exposed %d of %d accounts", len(accounts), accountCount) + } + } +} + +func TestCodexStoreRollbackOwnsAnOrdinaryReplacementOfAnActiveBatchAccount(t *testing.T) { + store := CodexStore{Dir: t.TempDir()} + staged := StoredCodexAccount{ + Email: "migrated@example.com", + Provider: ProviderCodex, + Auth: CodexAuthFile{ + AuthMode: "apikey", + OpenAIAPIKey: "sk-migrated", + }, + } + const batchID = "replacement-ownership" + if err := store.StageMigrationBatch(batchID, []StoredCodexAccount{staged}); err != nil { + t.Fatal(err) + } + if err := store.ActivateMigrationBatch(batchID, []string{staged.Email}); err != nil { + t.Fatal(err) + } + replacement := staged + replacement.MigrationBatchID = "" + replacement.Auth.OpenAIAPIKey = "sk-repaired" + if err := store.SaveStored(replacement); err != nil { + t.Fatal(err) + } + if err := store.RollbackMigrationBatch(batchID); err != nil { + t.Fatal(err) + } + accounts, err := store.ListStored() + if err != nil { + t.Fatal(err) + } + if len(accounts) != 0 { + t.Fatalf("rollback left ordinary replacement active: %+v", accounts) + } +} + +func TestCodexStoreRollbackIsAtomicForConcurrentReaders(t *testing.T) { + store := CodexStore{Dir: t.TempDir()} + const accountCount = 24 + const batchID = "rollback-visibility" + staged := make([]StoredCodexAccount, 0, accountCount) + ids := make([]string, 0, accountCount) + for i := 0; i < accountCount; i++ { + id := fmt.Sprintf("rollback-%03d@example.com", i) + ids = append(ids, id) + staged = append(staged, StoredCodexAccount{ + Email: id, + Provider: ProviderCodex, + Auth: CodexAuthFile{ + AuthMode: "apikey", + OpenAIAPIKey: "sk-test", + }, + }) + } + if err := store.StageMigrationBatch(batchID, staged); err != nil { + t.Fatal(err) + } + + stop := make(chan struct{}) + done := make(chan struct{}) + errors := make(chan error, 1) + go func() { + defer close(done) + for { + select { + case <-stop: + return + default: + } + if err := store.ActivateMigrationBatch(batchID, ids); err != nil { + errors <- err + return + } + if err := store.RollbackMigrationBatch(batchID); err != nil { + errors <- err + return + } + if err := store.StageMigrationBatch(batchID, staged); err != nil { + errors <- err + return + } + time.Sleep(time.Millisecond) + } + }() + defer func() { + close(stop) + <-done + }() + + for i := 0; i < 200; i++ { + select { + case err := <-errors: + t.Fatal(err) + default: + } + accounts, err := store.ListStored() + if err != nil { + t.Fatal(err) + } + if len(accounts) != 0 && len(accounts) != accountCount { + t.Fatalf("concurrent rollback list exposed %d of %d accounts", len(accounts), accountCount) + } + } +} + func TestCodexStoreRejectsAccountIdentifierThatWouldCreateHiddenState(t *testing.T) { store := CodexStore{Dir: t.TempDir()} account := StoredCodexAccount{ diff --git a/internal/proxy/multitenant_test.go b/internal/proxy/multitenant_test.go index dff9827e1..466e1f373 100644 --- a/internal/proxy/multitenant_test.go +++ b/internal/proxy/multitenant_test.go @@ -1327,6 +1327,137 @@ func TestTenantAccountUploadPreservesDistinctMigrationIDsAndLabels(t *testing.T) } } +func TestTenantAccountTextRejectsControlsAndUsesUTF8ByteLimits(t *testing.T) { + base := tenantAccountUpload{ + Provider: "openai-apikey", + AccountID: "legacy-account", + Label: "Shared account", + APIKey: "sk-test", + } + for name, mutate := range map[string]func(*tenantAccountUpload){ + "label control": func(input *tenantAccountUpload) { + input.Label = "unsafe\x1b[31m" + }, + "id control": func(input *tenantAccountUpload) { + input.AccountID = "unsafe\naccount" + }, + "label over 320 bytes": func(input *tenantAccountUpload) { + input.Label = strings.Repeat("é", 161) + }, + "id over 320 bytes": func(input *tenantAccountUpload) { + input.AccountID = strings.Repeat("é", 161) + }, + } { + t.Run(name, func(t *testing.T) { + input := base + mutate(&input) + if _, err := storedTenantMigrationAccount(input); err == nil { + t.Fatal("unsafe migration account text was accepted") + } + }) + } + + boundary := base + boundary.AccountID = strings.Repeat("é", 160) + boundary.Label = strings.Repeat("é", 160) + if _, err := storedTenantMigrationAccount(boundary); err != nil { + t.Fatalf("320-byte migration text rejected: %v", err) + } + + registry, handler, _ := newMultiTenantFixture(t) + _, key, err := registry.Create("team") + if err != nil { + t.Fatal(err) + } + body, err := json.Marshal(map[string]string{ + "provider": "openai-apikey", + "accountId": "safe-account", + "label": "unsafe\x1b[31m", + "apiKey": "sk-test", + }) + if err != nil { + t.Fatal(err) + } + response := httptest.NewRecorder() + handler.ServeHTTP( + response, + httptest.NewRequest( + http.MethodPost, + "/t/"+key+"/_subrouter/accounts", + strings.NewReader(string(body)), + ), + ) + if response.Code != http.StatusBadRequest { + t.Fatalf("ordinary upload status = %d, body = %s", response.Code, response.Body.String()) + } +} + +func TestTenantMigrationBatchStaysInactiveUntilAtomicActivation(t *testing.T) { + registry, handler, _ := newMultiTenantFixture(t) + _, key, err := registry.Create("team") + if err != nil { + t.Fatal(err) + } + basePath := "/t/" + key + "/_subrouter/accounts" + request := func(path, body string) *httptest.ResponseRecorder { + t.Helper() + response := httptest.NewRecorder() + handler.ServeHTTP( + response, + httptest.NewRequest(http.MethodPost, path, strings.NewReader(body)), + ) + return response + } + stage := request(basePath+"/migration/stage", `{ + "migrationId":"legacy-team", + "accounts":[{ + "provider":"codex", + "accountId":"legacy-account", + "label":"Shared account", + "tokens":{ + "accessToken":"access", + "refreshToken":"refresh", + "idToken":"id", + "accountID":"provider" + } + }] + }`) + if stage.Code != http.StatusOK { + t.Fatalf("stage status = %d, body = %s", stage.Code, stage.Body.String()) + } + + list := httptest.NewRecorder() + handler.ServeHTTP(list, httptest.NewRequest(http.MethodGet, basePath, nil)) + if list.Code != http.StatusOK || strings.TrimSpace(list.Body.String()) != "[]" { + t.Fatalf("staged account became active: %d %s", list.Code, list.Body.String()) + } + + activate := request(basePath+"/migration/activate", `{ + "migrationId":"legacy-team", + "accountIds":["legacy-account"] + }`) + if activate.Code != http.StatusOK { + t.Fatalf("activate status = %d, body = %s", activate.Code, activate.Body.String()) + } + list = httptest.NewRecorder() + handler.ServeHTTP(list, httptest.NewRequest(http.MethodGet, basePath, nil)) + if list.Code != http.StatusOK || !strings.Contains(list.Body.String(), "legacy-account") { + t.Fatalf("activated account missing: %d %s", list.Code, list.Body.String()) + } + + rollback := request(basePath+"/migration/rollback", `{ + "migrationId":"legacy-team" + }`) + if rollback.Code != http.StatusOK { + t.Fatalf("rollback status = %d, body = %s", rollback.Code, rollback.Body.String()) + } + list = httptest.NewRecorder() + handler.ServeHTTP(list, httptest.NewRequest(http.MethodGet, basePath, nil)) + if list.Code != http.StatusOK || strings.TrimSpace(list.Body.String()) != "[]" { + t.Fatalf("rolled-back account remained active: %d %s", list.Code, list.Body.String()) + } +} + func TestStackLoginRejectsInvalidTokenAndMismatchedTeam(t *testing.T) { registry := tenant.NewRegistry(t.TempDir()) base := Server{MaxBodyBytes: 1024} From 0e75818a1ecb6f8b279e73c5567cacad43582f84 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 3 Aug 2026 12:54:48 -0700 Subject: [PATCH 08/15] fix: stage hosted credentials before cutover --- cloudflare/packages/worker/src/index.ts | 40 +++- .../packages/worker/src/legacy-migration.ts | 177 ++++++++++++--- internal/accounts/codex_store.go | 206 +++++++++++++++++- internal/proxy/multitenant.go | 156 +++++++++++++ 4 files changed, 533 insertions(+), 46 deletions(-) diff --git a/cloudflare/packages/worker/src/index.ts b/cloudflare/packages/worker/src/index.ts index c30f82055..35692c47a 100644 --- a/cloudflare/packages/worker/src/index.ts +++ b/cloudflare/packages/worker/src/index.ts @@ -48,6 +48,7 @@ import { } from "./telemetry.ts" import { migrateLegacyTenant, + rollbackLegacyMigrationDestination, type LegacyMigrationAccount, } from "./legacy-migration.ts" @@ -281,6 +282,7 @@ interface HostedMigrationStateRow { readonly [key: string]: SqlStorageValue readonly org_id: string readonly accounts_json: string + readonly phase: string readonly started_at: number } @@ -1810,6 +1812,7 @@ export class SubrouterDurableObject extends DurableObject { CREATE TABLE IF NOT EXISTS hosted_migration_state( org_id TEXT PRIMARY KEY, accounts_json TEXT NOT NULL, + phase TEXT NOT NULL, started_at INTEGER NOT NULL ); INSERT OR IGNORE INTO subrouter_status(id) VALUES (1); @@ -2547,18 +2550,22 @@ export class SubrouterDurableObject extends DurableObject { readonly allowLoopback: boolean }): Promise<{ migrated: number }> { const orgId = this.resolveOrgId(input.orgId) + const migrationId = `legacy-${orgId}` const migrate = async (): Promise<{ migrated: number }> => ({ migrated: await migrateLegacyTenant({ destinationUrl: input.destinationUrl, tenantKey: input.tenantKey, finalizeSource: input.finalizeSource, allowLoopback: input.allowLoopback, + migrationId, source: { list: async () => this.listMigrationAccounts(orgId), begin: async () => this.beginMigrationAccounts(orgId), complete: async (accounts) => this.completeMigrationAccounts(orgId, accounts), restore: async () => this.restoreMigrationAccounts(orgId), + markActivating: async () => + this.markMigrationDestinationActivating(orgId), }, }), }) @@ -2571,6 +2578,18 @@ export class SubrouterDurableObject extends DurableObject { // single-use refresh token. No source row changes until they settle. this.migrationPreparing = true try { + const pending = this.hostedMigrationState(orgId) + if (pending?.phase === "activating") { + await rollbackLegacyMigrationDestination({ + destinationUrl: input.destinationUrl, + tenantKey: input.tenantKey, + migrationId, + allowLoopback: input.allowLoopback, + }) + this.restoreMigrationAccounts(orgId) + } else if (pending) { + this.restoreMigrationAccounts(orgId) + } await Promise.allSettled([...this.refreshInFlight.values()]) // Cloudflare bounds this barrier at 30 seconds. Migration uploads run in // parallel with a 10-second request deadline, leaving room for the two @@ -2621,8 +2640,9 @@ export class SubrouterDurableObject extends DurableObject { throw new Error("hosted migration recovery is pending") } this.ctx.storage.sql.exec( - `INSERT INTO hosted_migration_state(org_id, accounts_json, started_at) - VALUES (?, ?, ?)`, + `INSERT INTO hosted_migration_state( + org_id, accounts_json, phase, started_at + ) VALUES (?, ?, 'quiesced', ?)`, orgId, accountsJSON, Date.now() @@ -2657,6 +2677,21 @@ export class SubrouterDurableObject extends DurableObject { }) } + private markMigrationDestinationActivating(orgId: string): void { + this.ctx.storage.transactionSync(() => { + const state = this.hostedMigrationState(orgId) + if (!state || state.phase !== "quiesced") { + throw new Error("hosted migration source is not quiesced") + } + this.ctx.storage.sql.exec( + `UPDATE hosted_migration_state + SET phase = 'activating' + WHERE org_id = ?`, + orgId + ) + }) + } + private completeMigrationAccounts( orgId: string, accounts: ReadonlyArray @@ -3356,6 +3391,7 @@ export class SubrouterDurableObject extends DurableObject { if (states.length === 0) return this.ctx.storage.transactionSync(() => { for (const state of states) { + if (state.phase !== "quiesced") continue const accounts = JSON.parse( state.accounts_json ) as ReadonlyArray diff --git a/cloudflare/packages/worker/src/legacy-migration.ts b/cloudflare/packages/worker/src/legacy-migration.ts index 75bc3cae5..a78a1336a 100644 --- a/cloudflare/packages/worker/src/legacy-migration.ts +++ b/cloudflare/packages/worker/src/legacy-migration.ts @@ -19,6 +19,7 @@ export interface LegacyMigrationSource { readonly restore: ( accounts: ReadonlyArray ) => Promise + readonly markActivating?: () => Promise } const productionOrigins = new Set([ @@ -26,6 +27,7 @@ const productionOrigins = new Set([ "https://sr.cmux.dev", ]) const tenantKeyPattern = /^srt_[0-9a-f]{32}$/ +const migrationIdPattern = /^[a-z0-9][a-z0-9._-]{0,159}$/ const maxMigrationAccounts = 16 export async function migrateLegacyTenant(options: { @@ -33,6 +35,7 @@ export async function migrateLegacyTenant(options: { readonly tenantKey: unknown readonly finalizeSource: boolean readonly allowLoopback?: boolean + readonly migrationId?: string readonly source: LegacyMigrationSource readonly fetch?: typeof fetch }): Promise { @@ -42,8 +45,11 @@ export async function migrateLegacyTenant(options: { options.allowLoopback ?? false ) const tenantKey = migrationTenantKey(options.tenantKey) + const migrationId = normalizedMigrationID(options.migrationId) let accounts: ReadonlyArray = [] let sourceBegan = false + let destinationAttempted = false + let sourceCompleted = false try { if (options.finalizeSource) { // Mark the attempt before calling begin so a source with persisted @@ -53,19 +59,44 @@ export async function migrateLegacyTenant(options: { } else { accounts = await options.source.list() } + destinationAttempted = true const migrated = await migrateLegacyAccountsToHosted({ destinationUrl, tenantKey, + migrationId, accounts, allowLoopback: options.allowLoopback, fetch: options.fetch, }) if (sourceBegan) { + await options.source.markActivating?.() + await activateLegacyMigrationDestination({ + destinationUrl, + tenantKey, + migrationId, + accountIds: accounts.map((account) => account.id), + allowLoopback: options.allowLoopback, + fetch: options.fetch, + }) await options.source.complete(accounts) + sourceCompleted = true } return migrated } catch (error) { - if (options.finalizeSource) { + if (destinationAttempted && !sourceCompleted) { + try { + await rollbackLegacyMigrationDestination({ + destinationUrl, + tenantKey, + migrationId, + allowLoopback: options.allowLoopback, + fetch: options.fetch, + }) + } catch { + throw new Error("hosted migration failed and destination rollback failed") + } + } + if (sourceBegan && !sourceCompleted) { try { await options.source.restore(accounts) } catch { @@ -79,6 +110,7 @@ export async function migrateLegacyTenant(options: { export async function migrateLegacyAccountsToHosted(options: { readonly destinationUrl: unknown readonly tenantKey: unknown + readonly migrationId?: string readonly accounts: ReadonlyArray readonly allowLoopback?: boolean readonly fetch?: typeof fetch @@ -88,6 +120,7 @@ export async function migrateLegacyAccountsToHosted(options: { options.allowLoopback ?? false ) const tenantKey = migrationTenantKey(options.tenantKey) + const migrationId = normalizedMigrationID(options.migrationId) if (options.accounts.length > maxMigrationAccounts) { throw new Error("hosted migration account limit exceeded") } @@ -95,46 +128,122 @@ export async function migrateLegacyAccountsToHosted(options: { // Validate every source row before the first external mutation. A retry can // safely overwrite the same destination ids after a network interruption. const uploads = options.accounts.map(migrationUpload) - const fetchImpl = options.fetch ?? fetch - const results = await Promise.allSettled(uploads.map(async (upload) => { - let response: Response - try { - response = await fetchImpl(`${destination}/_subrouter/accounts`, { + const accountIds = uploads.map((upload) => String(upload.accountId)) + const body = await migrationDestinationRequest({ + destination, + tenantKey, + path: "/_subrouter/accounts/migration/stage", + body: { migrationId, accounts: uploads }, + fetch: options.fetch, + operation: "stage", + }) + if ( + body["ok"] !== true || + !Array.isArray(body["accountIds"]) || + body["accountIds"].length !== accountIds.length || + body["accountIds"].some( + (accountId, index) => accountId !== accountIds[index] + ) + ) { + throw new Error("hosted migration returned an invalid staged batch") + } + return uploads.length +} + +async function activateLegacyMigrationDestination(options: { + readonly destinationUrl: unknown + readonly tenantKey: unknown + readonly migrationId?: string + readonly accountIds: ReadonlyArray + readonly allowLoopback?: boolean + readonly fetch?: typeof fetch +}): Promise { + const body = await migrationDestinationRequest({ + destination: migrationDestination( + options.destinationUrl, + options.allowLoopback ?? false + ), + tenantKey: migrationTenantKey(options.tenantKey), + path: "/_subrouter/accounts/migration/activate", + body: { + migrationId: normalizedMigrationID(options.migrationId), + accountIds: options.accountIds, + }, + fetch: options.fetch, + operation: "activation", + }) + if (body["ok"] !== true) { + throw new Error("hosted migration returned an invalid activation") + } +} + +export async function rollbackLegacyMigrationDestination(options: { + readonly destinationUrl: unknown + readonly tenantKey: unknown + readonly migrationId?: string + readonly allowLoopback?: boolean + readonly fetch?: typeof fetch +}): Promise { + const body = await migrationDestinationRequest({ + destination: migrationDestination( + options.destinationUrl, + options.allowLoopback ?? false + ), + tenantKey: migrationTenantKey(options.tenantKey), + path: "/_subrouter/accounts/migration/rollback", + body: { migrationId: normalizedMigrationID(options.migrationId) }, + fetch: options.fetch, + operation: "rollback", + }) + if (body["ok"] !== true) { + throw new Error("hosted migration returned an invalid rollback") + } +} + +async function migrationDestinationRequest(options: { + readonly destination: string + readonly tenantKey: string + readonly path: string + readonly body: Record + readonly fetch?: typeof fetch + readonly operation: string +}): Promise> { + let response: Response + try { + response = await (options.fetch ?? fetch)( + `${options.destination}${options.path}`, + { method: "POST", redirect: "manual", signal: AbortSignal.timeout(10_000), headers: { - authorization: `Bearer ${tenantKey}`, + authorization: `Bearer ${options.tenantKey}`, "content-type": "application/json", }, - body: JSON.stringify(upload), - }) - } catch { - throw new Error("hosted migration destination is unavailable") - } - if (!response.ok) { - throw new Error(`hosted migration upload failed (${response.status})`) - } - const body = await response.json().catch(() => null) - if ( - !body || - typeof body !== "object" || - !("account" in body) || - !body.account || - typeof body.account !== "object" || - !("id" in body.account) || - body.account.id !== upload.accountId - ) { - throw new Error("hosted migration returned an invalid account") - } - })) - const failed = results.find( - (result): result is PromiseRejectedResult => result.status === "rejected" - ) - if (failed) { - throw failed.reason + body: JSON.stringify(options.body), + } + ) + } catch { + throw new Error("hosted migration destination is unavailable") } - return uploads.length + if (!response.ok) { + throw new Error( + `hosted migration ${options.operation} failed (${response.status})` + ) + } + const body = await response.json().catch(() => null) + if (!body || typeof body !== "object" || Array.isArray(body)) { + throw new Error("hosted migration returned an invalid response") + } + return body as Record +} + +function normalizedMigrationID(value: string | undefined): string { + const migrationId = value ?? "legacy-migration" + if (!migrationIdPattern.test(migrationId)) { + throw new Error("hosted migration id is invalid") + } + return migrationId } function migrationTenantKey(value: unknown): string { diff --git a/internal/accounts/codex_store.go b/internal/accounts/codex_store.go index 52a145c4f..67bf4764a 100644 --- a/internal/accounts/codex_store.go +++ b/internal/accounts/codex_store.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "path/filepath" + "slices" "sort" "strings" "time" @@ -27,15 +28,16 @@ func (e *StorageKeyCollisionError) Error() string { } type StoredCodexAccount struct { - Email string `json:"email"` - Label string `json:"label,omitempty"` - Provider Provider `json:"provider,omitempty"` - AddedAt string `json:"addedAt"` - Auth CodexAuthFile `json:"auth"` - ProjectID string `json:"projectId,omitempty"` - ProjectName string `json:"projectName,omitempty"` - AdminKeyLabel string `json:"adminKeyLabel,omitempty"` - Breadcrumbs []CodexAuthBreadcrumb `json:"breadcrumbs,omitempty"` + Email string `json:"email"` + Label string `json:"label,omitempty"` + Provider Provider `json:"provider,omitempty"` + MigrationBatchID string `json:"migrationBatchId,omitempty"` + AddedAt string `json:"addedAt"` + Auth CodexAuthFile `json:"auth"` + ProjectID string `json:"projectId,omitempty"` + ProjectName string `json:"projectName,omitempty"` + AdminKeyLabel string `json:"adminKeyLabel,omitempty"` + Breadcrumbs []CodexAuthBreadcrumb `json:"breadcrumbs,omitempty"` } type CodexAuthFile struct { @@ -125,6 +127,10 @@ func (s CodexStore) List() ([]Account, error) { } func (s CodexStore) ListStored() ([]StoredCodexAccount, error) { + return s.listStored(false) +} + +func (s CodexStore) listStored(includeInactiveMigrations bool) ([]StoredCodexAccount, error) { files, err := os.ReadDir(s.Dir) if errors.Is(err, os.ErrNotExist) { return nil, nil @@ -149,6 +155,15 @@ func (s CodexStore) ListStored() ([]StoredCodexAccount, error) { if err := json.Unmarshal(body, &stored); err != nil { return nil, fmt.Errorf("parse %s: %w", path, err) } + if !includeInactiveMigrations && stored.MigrationBatchID != "" { + active, err := s.migrationBatchActive(stored.MigrationBatchID) + if err != nil { + return nil, err + } + if !active { + continue + } + } if strings.TrimSpace(stored.Email) != "" { accounts = append(accounts, stored) } @@ -235,7 +250,7 @@ func (s CodexStore) saveStoredUnlocked(account StoredCodexAccount) error { if err := validateStoredAccountIdentifier(account.Email); err != nil { return err } - stored, err := s.ListStored() + stored, err := s.listStored(true) if err != nil { return err } @@ -264,6 +279,18 @@ func (s CodexStore) saveStoredUnlocked(account StoredCodexAccount) error { ExistingIdentifier: existing.Email, } } + if existing.MigrationBatchID != account.MigrationBatchID { + if account.MigrationBatchID != "" { + return fmt.Errorf("account %q belongs to a different migration state", account.Email) + } + active, activeErr := s.migrationBatchActive(existing.MigrationBatchID) + if activeErr != nil { + return activeErr + } + if existing.MigrationBatchID == "" || !active { + return fmt.Errorf("account %q belongs to a different migration state", account.Email) + } + } } else if !errors.Is(err, os.ErrNotExist) { return err } @@ -304,6 +331,15 @@ func (s CodexStore) FindStored(identifier string) (StoredCodexAccount, bool, err return StoredCodexAccount{}, false, err } if strings.EqualFold(strings.TrimSpace(account.Email), needle) { + if account.MigrationBatchID != "" { + active, activeErr := s.migrationBatchActive(account.MigrationBatchID) + if activeErr != nil { + return StoredCodexAccount{}, false, activeErr + } + if !active { + return StoredCodexAccount{}, false, nil + } + } return account, true, nil } } else if !errors.Is(err, os.ErrNotExist) { @@ -340,6 +376,156 @@ func (s CodexStore) FindStored(identifier string) (StoredCodexAccount, bool, err return matches[0], true, nil } +// StageMigrationBatch persists credentials outside every routing and refresh +// path. A single activation marker later makes the complete batch visible. +func (s CodexStore) StageMigrationBatch(batchID string, staged []StoredCodexAccount) error { + if err := validateMigrationBatchID(batchID); err != nil { + return err + } + lock, err := s.lockStoredAccount("migration-batch") + if err != nil { + return err + } + defer lock.Close() + if active, err := s.migrationBatchActive(batchID); err != nil { + return err + } else if active { + return errors.New("migration batch is already active") + } + desired := make(map[string]bool, len(staged)) + for i := range staged { + if err := validateStoredAccountIdentifier(staged[i].Email); err != nil { + return err + } + key := strings.ToLower(strings.TrimSpace(staged[i].Email)) + if desired[key] { + return fmt.Errorf("duplicate migration account %q", staged[i].Email) + } + desired[key] = true + staged[i].MigrationBatchID = batchID + if err := s.saveStoredUnlocked(staged[i]); err != nil { + return err + } + } + all, err := s.listStored(true) + if err != nil { + return err + } + for _, account := range all { + if account.MigrationBatchID != batchID || desired[strings.ToLower(strings.TrimSpace(account.Email))] { + continue + } + if err := os.Remove(account.SourcePath(s)); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + } + return nil +} + +// ActivateMigrationBatch atomically publishes a previously staged batch. +func (s CodexStore) ActivateMigrationBatch(batchID string, expectedIDs []string) error { + if err := validateMigrationBatchID(batchID); err != nil { + return err + } + lock, err := s.lockStoredAccount("migration-batch") + if err != nil { + return err + } + defer lock.Close() + all, err := s.listStored(true) + if err != nil { + return err + } + actual := make([]string, 0, len(expectedIDs)) + for _, account := range all { + if account.MigrationBatchID == batchID { + actual = append(actual, account.Email) + } + } + sort.Strings(actual) + expected := append([]string(nil), expectedIDs...) + for i := range expected { + expected[i] = strings.TrimSpace(expected[i]) + if err := validateStoredAccountIdentifier(expected[i]); err != nil { + return err + } + } + sort.Strings(expected) + if !slices.Equal(actual, expected) { + return errors.New("migration batch account set does not match staged credentials") + } + body, err := json.Marshal(map[string]any{"accountIds": expected}) + if err != nil { + return err + } + return writeFileAtomic(s.migrationBatchMarker(batchID), append(body, '\n'), 0o600) +} + +// RollbackMigrationBatch deactivates and removes only credentials owned by the +// named batch. It is idempotent so callers can resolve an ambiguous activation. +func (s CodexStore) RollbackMigrationBatch(batchID string) error { + if err := validateMigrationBatchID(batchID); err != nil { + return err + } + lock, err := s.lockStoredAccount("migration-batch") + if err != nil { + return err + } + defer lock.Close() + if err := os.Remove(s.migrationBatchMarker(batchID)); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + all, err := s.listStored(true) + if err != nil { + return err + } + for _, account := range all { + if account.MigrationBatchID != batchID { + continue + } + if err := os.Remove(account.SourcePath(s)); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + } + return nil +} + +func validateMigrationBatchID(batchID string) error { + if batchID == "" || len(batchID) > 160 { + return errors.New("migration id is invalid") + } + for _, character := range batchID { + if (character >= 'a' && character <= 'z') || + (character >= '0' && character <= '9') || + character == '-' || character == '_' || character == '.' { + continue + } + return errors.New("migration id is invalid") + } + return nil +} + +func (s CodexStore) migrationBatchMarker(batchID string) string { + return filepath.Join(s.Dir, ".migration-batches", batchID+".active.json") +} + +func (s CodexStore) migrationBatchActive(batchID string) (bool, error) { + if batchID == "" { + return false, nil + } + if err := validateMigrationBatchID(batchID); err != nil { + return false, err + } + _, err := os.Stat(s.migrationBatchMarker(batchID)) + if err == nil { + return true, nil + } + if errors.Is(err, os.ErrNotExist) { + return false, nil + } + return false, err +} + func (s CodexStore) RemoveStored(identifier string) (StoredCodexAccount, bool, error) { account, ok, err := s.FindStored(identifier) if err != nil || !ok { diff --git a/internal/proxy/multitenant.go b/internal/proxy/multitenant.go index 436a7ceff..712f92355 100644 --- a/internal/proxy/multitenant.go +++ b/internal/proxy/multitenant.go @@ -323,6 +323,18 @@ func tenantScopedHandler(server Server, t tenant.Tenant) http.Handler { return } } + if r.URL.Path == "/_subrouter/accounts/migration/stage" && r.Method == http.MethodPost { + handleTenantMigrationStage(&server, w, r) + return + } + if r.URL.Path == "/_subrouter/accounts/migration/activate" && r.Method == http.MethodPost { + handleTenantMigrationActivate(&server, w, r) + return + } + if r.URL.Path == "/_subrouter/accounts/migration/rollback" && r.Method == http.MethodPost { + handleTenantMigrationRollback(&server, w, r) + return + } if strings.HasPrefix(r.URL.Path, "/_subrouter/accounts/") && r.Method == http.MethodDelete { handleTenantAccountDelete(&server, w, r) return @@ -687,6 +699,150 @@ type tenantAccountUpload struct { ClaudeAIOAuth *agentclaude.CredentialInfo `json:"claudeAiOauth"` } +type tenantMigrationStageInput struct { + MigrationID string `json:"migrationId"` + Accounts []tenantAccountUpload `json:"accounts"` +} + +type tenantMigrationBatchInput struct { + MigrationID string `json:"migrationId"` + AccountIDs []string `json:"accountIds,omitempty"` +} + +func handleTenantMigrationStage(server *Server, w http.ResponseWriter, r *http.Request) { + if server.AccountRef == nil { + http.Error(w, "tenant account store unavailable", http.StatusServiceUnavailable) + return + } + var input tenantMigrationStageInput + if !decodeTenantMigrationJSON(server, w, r, &input) { + return + } + if len(input.Accounts) == 0 || len(input.Accounts) > 16 { + http.Error(w, "migration account count is invalid", http.StatusBadRequest) + return + } + staged := make([]accounts.StoredCodexAccount, 0, len(input.Accounts)) + ids := make([]string, 0, len(input.Accounts)) + for _, upload := range input.Accounts { + account, err := storedTenantMigrationAccount(upload) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + staged = append(staged, account) + ids = append(ids, account.Email) + } + if err := server.AccountRef.store.StageMigrationBatch(strings.TrimSpace(input.MigrationID), staged); err != nil { + http.Error(w, "stage migration accounts", http.StatusConflict) + return + } + writeJSON(w, map[string]any{"ok": true, "accountIds": ids}) +} + +func handleTenantMigrationActivate(server *Server, w http.ResponseWriter, r *http.Request) { + if server.AccountRef == nil { + http.Error(w, "tenant account store unavailable", http.StatusServiceUnavailable) + return + } + var input tenantMigrationBatchInput + if !decodeTenantMigrationJSON(server, w, r, &input) { + return + } + batchID := strings.TrimSpace(input.MigrationID) + if err := server.AccountRef.store.ActivateMigrationBatch(batchID, input.AccountIDs); err != nil { + http.Error(w, "activate migration accounts", http.StatusConflict) + return + } + if _, _, err := server.reloadAccounts(r.Context()); err != nil { + _ = server.AccountRef.store.RollbackMigrationBatch(batchID) + _, _, _ = server.reloadAccounts(r.Context()) + http.Error(w, "activate migration accounts", http.StatusInternalServerError) + return + } + writeJSON(w, map[string]any{"ok": true, "activated": input.AccountIDs}) +} + +func handleTenantMigrationRollback(server *Server, w http.ResponseWriter, r *http.Request) { + if server.AccountRef == nil { + http.Error(w, "tenant account store unavailable", http.StatusServiceUnavailable) + return + } + var input tenantMigrationBatchInput + if !decodeTenantMigrationJSON(server, w, r, &input) { + return + } + if err := server.AccountRef.store.RollbackMigrationBatch(strings.TrimSpace(input.MigrationID)); err != nil { + http.Error(w, "rollback migration accounts", http.StatusInternalServerError) + return + } + if _, _, err := server.reloadAccounts(r.Context()); err != nil { + http.Error(w, "rollback migration accounts", http.StatusInternalServerError) + return + } + writeJSON(w, map[string]any{"ok": true, "rolledBack": true}) +} + +func decodeTenantMigrationJSON(server *Server, w http.ResponseWriter, r *http.Request, output any) bool { + bodyLimit := server.MaxBodyBytes + if bodyLimit <= 0 { + bodyLimit = 1 << 20 + } + decoder := json.NewDecoder(io.LimitReader(r.Body, bodyLimit)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(output); err != nil { + http.Error(w, "invalid JSON body", http.StatusBadRequest) + return false + } + if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) { + http.Error(w, "invalid JSON body", http.StatusBadRequest) + return false + } + return true +} + +func storedTenantMigrationAccount(input tenantAccountUpload) (accounts.StoredCodexAccount, error) { + input.Provider = strings.ToLower(strings.TrimSpace(input.Provider)) + input.AccountID = strings.TrimSpace(input.AccountID) + input.Label = strings.TrimSpace(input.Label) + if input.Label == "" || len(input.Label) > 320 { + return accounts.StoredCodexAccount{}, errors.New("account label is required") + } + if input.AccountID == "" || len(input.AccountID) > 320 || containsTerminalControl(input.AccountID) { + return accounts.StoredCodexAccount{}, errors.New("account id is invalid") + } + switch input.Provider { + case "codex": + if input.Tokens == nil || input.Tokens.AccessToken == "" || input.Tokens.RefreshToken == "" || input.Tokens.IDToken == "" { + return accounts.StoredCodexAccount{}, errors.New("complete Codex OAuth tokens are required") + } + return accounts.StoredCodexAccount{ + Email: input.AccountID, Label: input.Label, Provider: accounts.ProviderCodex, + Auth: accounts.CodexAuthFile{ + AuthMode: "chatgpt", + Tokens: &accounts.CodexTokens{ + AccessToken: input.Tokens.AccessToken, RefreshToken: input.Tokens.RefreshToken, + IDToken: input.Tokens.IDToken, AccountID: input.Tokens.AccountID, + }, + }, + }, nil + case "openai-apikey", "anthropic-apikey": + if strings.TrimSpace(input.APIKey) == "" { + return accounts.StoredCodexAccount{}, errors.New("API key is required") + } + provider := accounts.ProviderCodex + if input.Provider == "anthropic-apikey" { + provider = accounts.ProviderClaude + } + return accounts.StoredCodexAccount{ + Email: input.AccountID, Label: input.Label, Provider: provider, + Auth: accounts.CodexAuthFile{AuthMode: "apikey", OpenAIAPIKey: strings.TrimSpace(input.APIKey)}, + }, nil + default: + return accounts.StoredCodexAccount{}, errors.New("unsupported migration provider") + } +} + func handleTenantAccountUpload(server *Server, w http.ResponseWriter, r *http.Request) { if server.AccountRef == nil { http.Error(w, "tenant account store unavailable", http.StatusServiceUnavailable) From 4a4fe6619c79db4352f24341dd1fa276b56cd055 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 3 Aug 2026 13:09:15 -0700 Subject: [PATCH 09/15] fix: bind migration recovery and persist receipts --- cloudflare/packages/worker/src/index.ts | 98 ++++++++++++++++--- .../packages/worker/src/legacy-migration.ts | 28 ++++-- 2 files changed, 108 insertions(+), 18 deletions(-) diff --git a/cloudflare/packages/worker/src/index.ts b/cloudflare/packages/worker/src/index.ts index 35692c47a..5847bcb90 100644 --- a/cloudflare/packages/worker/src/index.ts +++ b/cloudflare/packages/worker/src/index.ts @@ -49,6 +49,7 @@ import { import { migrateLegacyTenant, rollbackLegacyMigrationDestination, + validatedLegacyMigrationBinding, type LegacyMigrationAccount, } from "./legacy-migration.ts" @@ -283,9 +284,19 @@ interface HostedMigrationStateRow { readonly org_id: string readonly accounts_json: string readonly phase: string + readonly destination_origin: string + readonly tenant_key_sha256: string + readonly migration_id: string + readonly account_count: number readonly started_at: number } +interface HostedMigrationBinding { + readonly destinationOrigin: string + readonly tenantKeySha256: string + readonly migrationId: string +} + interface TableInfoRow { readonly [key: string]: SqlStorageValue readonly name: string @@ -1813,6 +1824,10 @@ export class SubrouterDurableObject extends DurableObject { org_id TEXT PRIMARY KEY, accounts_json TEXT NOT NULL, phase TEXT NOT NULL, + destination_origin TEXT NOT NULL, + tenant_key_sha256 TEXT NOT NULL, + migration_id TEXT NOT NULL, + account_count INTEGER NOT NULL, started_at INTEGER NOT NULL ); INSERT OR IGNORE INTO subrouter_status(id) VALUES (1); @@ -2551,16 +2566,27 @@ export class SubrouterDurableObject extends DurableObject { }): Promise<{ migrated: number }> { const orgId = this.resolveOrgId(input.orgId) const migrationId = `legacy-${orgId}` + const validated = validatedLegacyMigrationBinding({ + destinationUrl: input.destinationUrl, + tenantKey: input.tenantKey, + migrationId, + allowLoopback: input.allowLoopback, + }) + const binding: HostedMigrationBinding = { + destinationOrigin: validated.destinationUrl, + tenantKeySha256: await tenantKeySha256Hex(validated.tenantKey), + migrationId: validated.migrationId, + } const migrate = async (): Promise<{ migrated: number }> => ({ migrated: await migrateLegacyTenant({ - destinationUrl: input.destinationUrl, - tenantKey: input.tenantKey, + destinationUrl: validated.destinationUrl, + tenantKey: validated.tenantKey, finalizeSource: input.finalizeSource, allowLoopback: input.allowLoopback, - migrationId, + migrationId: validated.migrationId, source: { list: async () => this.listMigrationAccounts(orgId), - begin: async () => this.beginMigrationAccounts(orgId), + begin: async () => this.beginMigrationAccounts(orgId, binding), complete: async (accounts) => this.completeMigrationAccounts(orgId, accounts), restore: async () => this.restoreMigrationAccounts(orgId), @@ -2570,6 +2596,20 @@ export class SubrouterDurableObject extends DurableObject { }), }) if (!input.finalizeSource) return await migrate() + const receipt = this.hostedMigrationState(orgId) + if (receipt?.phase === "completed") { + this.assertHostedMigrationBinding(receipt, binding) + const sourceAccounts = this.ctx.storage.sql + .exec( + "SELECT COUNT(*) AS count FROM accounts WHERE org_id = ?", + orgId + ) + .one().count + if (sourceAccounts !== 0) { + throw new Error("legacy source changed after hosted migration completed") + } + return { migrated: receipt.account_count } + } if (this.migrationPreparing) { throw new Error("hosted migration is already in progress") } @@ -2579,11 +2619,14 @@ export class SubrouterDurableObject extends DurableObject { this.migrationPreparing = true try { const pending = this.hostedMigrationState(orgId) + if (pending) { + this.assertHostedMigrationBinding(pending, binding) + } if (pending?.phase === "activating") { await rollbackLegacyMigrationDestination({ - destinationUrl: input.destinationUrl, - tenantKey: input.tenantKey, - migrationId, + destinationUrl: validated.destinationUrl, + tenantKey: validated.tenantKey, + migrationId: validated.migrationId, allowLoopback: input.allowLoopback, }) this.restoreMigrationAccounts(orgId) @@ -2625,7 +2668,8 @@ export class SubrouterDurableObject extends DurableObject { } private beginMigrationAccounts( - orgId: string + orgId: string, + binding: HostedMigrationBinding ): ReadonlyArray { const accounts = this.listMigrationAccounts(orgId) const accountsJSON = JSON.stringify(accounts) @@ -2641,10 +2685,21 @@ export class SubrouterDurableObject extends DurableObject { } this.ctx.storage.sql.exec( `INSERT INTO hosted_migration_state( - org_id, accounts_json, phase, started_at - ) VALUES (?, ?, 'quiesced', ?)`, + org_id, + accounts_json, + phase, + destination_origin, + tenant_key_sha256, + migration_id, + account_count, + started_at + ) VALUES (?, ?, 'quiesced', ?, ?, ?, ?, ?)`, orgId, accountsJSON, + binding.destinationOrigin, + binding.tenantKeySha256, + binding.migrationId, + accounts.length, Date.now() ) this.ctx.storage.sql.exec( @@ -2699,7 +2754,11 @@ export class SubrouterDurableObject extends DurableObject { this.ctx.storage.transactionSync(() => { const sql = this.ctx.storage.sql const state = this.hostedMigrationState(orgId) - if (!state || state.accounts_json !== JSON.stringify(accounts)) { + if ( + !state || + state.phase !== "activating" || + state.accounts_json !== JSON.stringify(accounts) + ) { throw new Error("hosted migration source snapshot is unavailable") } const rows = sql @@ -2720,7 +2779,9 @@ export class SubrouterDurableObject extends DurableObject { this.deleteAccountRows(sql, orgId, account.id) } sql.exec( - "DELETE FROM hosted_migration_state WHERE org_id = ?", + `UPDATE hosted_migration_state + SET accounts_json = '[]', phase = 'completed' + WHERE org_id = ?`, orgId ) }) @@ -3363,6 +3424,19 @@ export class SubrouterDurableObject extends DurableObject { .toArray()[0] ?? null } + private assertHostedMigrationBinding( + state: HostedMigrationStateRow, + binding: HostedMigrationBinding + ): void { + if ( + state.destination_origin !== binding.destinationOrigin || + state.tenant_key_sha256 !== binding.tenantKeySha256 || + state.migration_id !== binding.migrationId + ) { + throw new Error("hosted migration recovery destination does not match") + } + } + private migrationAccountMatchesRow( account: LegacyMigrationAccount | undefined, row: AccountRow diff --git a/cloudflare/packages/worker/src/legacy-migration.ts b/cloudflare/packages/worker/src/legacy-migration.ts index a78a1336a..7d98a4709 100644 --- a/cloudflare/packages/worker/src/legacy-migration.ts +++ b/cloudflare/packages/worker/src/legacy-migration.ts @@ -30,6 +30,26 @@ const tenantKeyPattern = /^srt_[0-9a-f]{32}$/ const migrationIdPattern = /^[a-z0-9][a-z0-9._-]{0,159}$/ const maxMigrationAccounts = 16 +export function validatedLegacyMigrationBinding(options: { + readonly destinationUrl: unknown + readonly tenantKey: unknown + readonly migrationId?: string + readonly allowLoopback?: boolean +}): { + readonly destinationUrl: string + readonly tenantKey: string + readonly migrationId: string +} { + return { + destinationUrl: migrationDestination( + options.destinationUrl, + options.allowLoopback ?? false + ), + tenantKey: migrationTenantKey(options.tenantKey), + migrationId: normalizedMigrationID(options.migrationId), + } +} + export async function migrateLegacyTenant(options: { readonly destinationUrl: unknown readonly tenantKey: unknown @@ -40,12 +60,8 @@ export async function migrateLegacyTenant(options: { readonly fetch?: typeof fetch }): Promise { // Reject operator input before quiescing any source credential. - const destinationUrl = migrationDestination( - options.destinationUrl, - options.allowLoopback ?? false - ) - const tenantKey = migrationTenantKey(options.tenantKey) - const migrationId = normalizedMigrationID(options.migrationId) + const { destinationUrl, tenantKey, migrationId } = + validatedLegacyMigrationBinding(options) let accounts: ReadonlyArray = [] let sourceBegan = false let destinationAttempted = false From 7a5b71e68c5ca62b2d4381455f9a952676e685ef Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 3 Aug 2026 13:26:03 -0700 Subject: [PATCH 10/15] fix: close hosted migration races --- cloudflare/packages/worker/src/index.ts | 12 ++ .../packages/worker/src/legacy-migration.ts | 6 +- internal/accounts/codex_store.go | 119 +++++++++++++++++- 3 files changed, 131 insertions(+), 6 deletions(-) diff --git a/cloudflare/packages/worker/src/index.ts b/cloudflare/packages/worker/src/index.ts index 5847bcb90..dc5ebe189 100644 --- a/cloudflare/packages/worker/src/index.ts +++ b/cloudflare/packages/worker/src/index.ts @@ -2428,6 +2428,7 @@ export class SubrouterDurableObject extends DurableObject { } async upsertAccount(input: UpsertAccountInput): Promise<{ account: StoredAccount }> { + this.assertLegacySourceWritable(input.orgId) const sql = this.ctx.storage.sql const now = Date.now() sql.exec( @@ -2484,6 +2485,7 @@ export class SubrouterDurableObject extends DurableObject { readonly accountId: string }): Promise<{ account: StoredAccount }> { const orgId = this.resolveOrgId(input.orgId) + this.assertLegacySourceWritable(orgId) const accountId = this.requireNonEmpty(input.accountId, "accountId") const row = this.getAccountRow( this.ctx.storage.sql, @@ -2789,6 +2791,7 @@ export class SubrouterDurableObject extends DurableObject { async deleteAccount(input: { readonly orgId?: string; readonly accountId: string }): Promise<{ ok: true }> { const orgId = this.resolveOrgId(input.orgId) + this.assertLegacySourceWritable(orgId) const accountId = this.requireNonEmpty(input.accountId, "accountId") const row = this.getAccountRow(this.ctx.storage.sql, orgId, accountId, false) if (!row) throw new Error("account not found") @@ -2803,6 +2806,7 @@ export class SubrouterDurableObject extends DurableObject { readonly replacement: UpsertAccountInput }): Promise<{ account: StoredAccount }> { const orgId = this.resolveOrgId(input.orgId) + this.assertLegacySourceWritable(orgId) const accountId = this.requireNonEmpty(input.accountId, "accountId") const existing = this.getAccountRow( this.ctx.storage.sql, @@ -3424,6 +3428,14 @@ export class SubrouterDurableObject extends DurableObject { .toArray()[0] ?? null } + private assertLegacySourceWritable(orgId: string): void { + if (this.hostedMigrationState(orgId)) { + throw new Error( + "legacy source is read-only during or after hosted migration" + ) + } + } + private assertHostedMigrationBinding( state: HostedMigrationStateRow, binding: HostedMigrationBinding diff --git a/cloudflare/packages/worker/src/legacy-migration.ts b/cloudflare/packages/worker/src/legacy-migration.ts index 7d98a4709..b0c227410 100644 --- a/cloudflare/packages/worker/src/legacy-migration.ts +++ b/cloudflare/packages/worker/src/legacy-migration.ts @@ -65,6 +65,7 @@ export async function migrateLegacyTenant(options: { let accounts: ReadonlyArray = [] let sourceBegan = false let destinationAttempted = false + let activationAttempted = false let sourceCompleted = false try { if (options.finalizeSource) { @@ -86,6 +87,7 @@ export async function migrateLegacyTenant(options: { }) if (sourceBegan) { await options.source.markActivating?.() + activationAttempted = true await activateLegacyMigrationDestination({ destinationUrl, tenantKey, @@ -109,7 +111,9 @@ export async function migrateLegacyTenant(options: { fetch: options.fetch, }) } catch { - throw new Error("hosted migration failed and destination rollback failed") + if (activationAttempted) { + throw new Error("hosted migration failed and destination rollback failed") + } } } if (sourceBegan && !sourceCompleted) { diff --git a/internal/accounts/codex_store.go b/internal/accounts/codex_store.go index 67bf4764a..a93d46dd2 100644 --- a/internal/accounts/codex_store.go +++ b/internal/accounts/codex_store.go @@ -18,6 +18,8 @@ type CodexStore struct { Dir string } +const migrationBatchControlLockID = ".subrouter-migration-batch-control" + type StorageKeyCollisionError struct { Identifier string ExistingIdentifier string @@ -238,6 +240,13 @@ func (s CodexStore) SaveStored(account StoredCodexAccount) error { if err := validateStoredAccountIdentifier(account.Email); err != nil { return err } + if account.MigrationBatchID != "" { + batchLock, err := s.lockStoredAccount(migrationBatchControlLockID) + if err != nil { + return err + } + defer batchLock.Close() + } lock, err := s.lockStoredAccount(account.Email) if err != nil { return err @@ -382,7 +391,7 @@ func (s CodexStore) StageMigrationBatch(batchID string, staged []StoredCodexAcco if err := validateMigrationBatchID(batchID); err != nil { return err } - lock, err := s.lockStoredAccount("migration-batch") + lock, err := s.lockStoredAccount(migrationBatchControlLockID) if err != nil { return err } @@ -393,6 +402,7 @@ func (s CodexStore) StageMigrationBatch(batchID string, staged []StoredCodexAcco return errors.New("migration batch is already active") } desired := make(map[string]bool, len(staged)) + accountIDs := make([]string, 0, len(staged)) for i := range staged { if err := validateStoredAccountIdentifier(staged[i].Email); err != nil { return err @@ -403,11 +413,28 @@ func (s CodexStore) StageMigrationBatch(batchID string, staged []StoredCodexAcco } desired[key] = true staged[i].MigrationBatchID = batchID + accountIDs = append(accountIDs, staged[i].Email) + } + all, err := s.listStored(true) + if err != nil { + return err + } + for _, account := range all { + if account.MigrationBatchID == batchID { + accountIDs = append(accountIDs, account.Email) + } + } + accountLocks, err := s.lockStoredAccounts(accountIDs) + if err != nil { + return err + } + defer closeAccountFileLocks(accountLocks) + for i := range staged { if err := s.saveStoredUnlocked(staged[i]); err != nil { return err } } - all, err := s.listStored(true) + all, err = s.listStored(true) if err != nil { return err } @@ -427,7 +454,7 @@ func (s CodexStore) ActivateMigrationBatch(batchID string, expectedIDs []string) if err := validateMigrationBatchID(batchID); err != nil { return err } - lock, err := s.lockStoredAccount("migration-batch") + lock, err := s.lockStoredAccount(migrationBatchControlLockID) if err != nil { return err } @@ -451,6 +478,22 @@ func (s CodexStore) ActivateMigrationBatch(batchID string, expectedIDs []string) } } sort.Strings(expected) + accountLocks, err := s.lockStoredAccounts(append(append([]string(nil), actual...), expected...)) + if err != nil { + return err + } + defer closeAccountFileLocks(accountLocks) + all, err = s.listStored(true) + if err != nil { + return err + } + actual = actual[:0] + for _, account := range all { + if account.MigrationBatchID == batchID { + actual = append(actual, account.Email) + } + } + sort.Strings(actual) if !slices.Equal(actual, expected) { return errors.New("migration batch account set does not match staged credentials") } @@ -467,15 +510,30 @@ func (s CodexStore) RollbackMigrationBatch(batchID string) error { if err := validateMigrationBatchID(batchID); err != nil { return err } - lock, err := s.lockStoredAccount("migration-batch") + lock, err := s.lockStoredAccount(migrationBatchControlLockID) if err != nil { return err } defer lock.Close() + all, err := s.listStored(true) + if err != nil { + return err + } + accountIDs := make([]string, 0, len(all)) + for _, account := range all { + if account.MigrationBatchID == batchID { + accountIDs = append(accountIDs, account.Email) + } + } + accountLocks, err := s.lockStoredAccounts(accountIDs) + if err != nil { + return err + } + defer closeAccountFileLocks(accountLocks) if err := os.Remove(s.migrationBatchMarker(batchID)); err != nil && !errors.Is(err, os.ErrNotExist) { return err } - all, err := s.listStored(true) + all, err = s.listStored(true) if err != nil { return err } @@ -505,6 +563,39 @@ func validateMigrationBatchID(batchID string) error { return nil } +func (s CodexStore) lockStoredAccounts(identifiers []string) ([]*accountFileLock, error) { + unique := make(map[string]struct{}, len(identifiers)) + canonical := make([]string, 0, len(identifiers)) + for _, identifier := range identifiers { + key := strings.ToLower(strings.TrimSpace(identifier)) + if key == "" { + continue + } + if _, exists := unique[key]; exists { + continue + } + unique[key] = struct{}{} + canonical = append(canonical, key) + } + sort.Strings(canonical) + locks := make([]*accountFileLock, 0, len(canonical)) + for _, identifier := range canonical { + lock, err := s.lockStoredAccount(identifier) + if err != nil { + closeAccountFileLocks(locks) + return nil, err + } + locks = append(locks, lock) + } + return locks, nil +} + +func closeAccountFileLocks(locks []*accountFileLock) { + for i := len(locks) - 1; i >= 0; i-- { + _ = locks[i].Close() + } +} + func (s CodexStore) migrationBatchMarker(batchID string) string { return filepath.Join(s.Dir, ".migration-batches", batchID+".active.json") } @@ -531,6 +622,15 @@ func (s CodexStore) RemoveStored(identifier string) (StoredCodexAccount, bool, e if err != nil || !ok { return account, ok, err } + lock, err := s.lockStoredAccount(account.Email) + if err != nil { + return account, false, err + } + defer lock.Close() + account, ok, err = s.FindStored(identifier) + if err != nil || !ok { + return account, ok, err + } if err := os.Remove(filepath.Join(s.Dir, emailToFilename(account.Email))); err != nil { return account, false, err } @@ -551,6 +651,15 @@ func (s CodexStore) MigrateStoredAway(identifier string) (string, bool, error) { if err != nil || !ok { return "", ok, err } + lock, err := s.lockStoredAccount(account.Email) + if err != nil { + return "", false, err + } + defer lock.Close() + account, ok, err = s.FindStored(identifier) + if err != nil || !ok { + return "", ok, err + } name := emailToFilename(account.Email) dest := filepath.Join(s.Dir, MigratedDirName) if err := os.MkdirAll(dest, 0o700); err != nil { From 46350f8e74693969f0e44ca22001547c05b5f9a1 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 3 Aug 2026 13:38:41 -0700 Subject: [PATCH 11/15] fix: retain staged migration recovery binding --- cloudflare/packages/worker/src/index.ts | 33 +++++++++++++------ .../packages/worker/src/legacy-migration.ts | 9 +++-- 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/cloudflare/packages/worker/src/index.ts b/cloudflare/packages/worker/src/index.ts index dc5ebe189..05e77e5ec 100644 --- a/cloudflare/packages/worker/src/index.ts +++ b/cloudflare/packages/worker/src/index.ts @@ -2591,7 +2591,8 @@ export class SubrouterDurableObject extends DurableObject { begin: async () => this.beginMigrationAccounts(orgId, binding), complete: async (accounts) => this.completeMigrationAccounts(orgId, accounts), - restore: async () => this.restoreMigrationAccounts(orgId), + restore: async (_accounts, options) => + this.restoreMigrationAccounts(orgId, options.preserveRecovery), markActivating: async () => this.markMigrationDestinationActivating(orgId), }, @@ -2624,7 +2625,7 @@ export class SubrouterDurableObject extends DurableObject { if (pending) { this.assertHostedMigrationBinding(pending, binding) } - if (pending?.phase === "activating") { + if (pending) { await rollbackLegacyMigrationDestination({ destinationUrl: validated.destinationUrl, tenantKey: validated.tenantKey, @@ -2632,8 +2633,6 @@ export class SubrouterDurableObject extends DurableObject { allowLoopback: input.allowLoopback, }) this.restoreMigrationAccounts(orgId) - } else if (pending) { - this.restoreMigrationAccounts(orgId) } await Promise.allSettled([...this.refreshInFlight.values()]) // Cloudflare bounds this barrier at 30 seconds. Migration uploads run in @@ -2712,7 +2711,10 @@ export class SubrouterDurableObject extends DurableObject { return accounts } - private restoreMigrationAccounts(orgId: string): void { + private restoreMigrationAccounts( + orgId: string, + preserveRecovery = false + ): void { this.ctx.storage.transactionSync(() => { const state = this.hostedMigrationState(orgId) if (!state) return @@ -2727,10 +2729,19 @@ export class SubrouterDurableObject extends DurableObject { orgId ) } - this.ctx.storage.sql.exec( - "DELETE FROM hosted_migration_state WHERE org_id = ?", - orgId - ) + if (preserveRecovery) { + this.ctx.storage.sql.exec( + `UPDATE hosted_migration_state + SET phase = 'cleanup_pending' + WHERE org_id = ?`, + orgId + ) + } else { + this.ctx.storage.sql.exec( + "DELETE FROM hosted_migration_state WHERE org_id = ?", + orgId + ) + } }) } @@ -3490,7 +3501,9 @@ export class SubrouterDurableObject extends DurableObject { ) } this.ctx.storage.sql.exec( - "DELETE FROM hosted_migration_state WHERE org_id = ?", + `UPDATE hosted_migration_state + SET phase = 'cleanup_pending' + WHERE org_id = ?`, state.org_id ) } diff --git a/cloudflare/packages/worker/src/legacy-migration.ts b/cloudflare/packages/worker/src/legacy-migration.ts index b0c227410..a90ade98a 100644 --- a/cloudflare/packages/worker/src/legacy-migration.ts +++ b/cloudflare/packages/worker/src/legacy-migration.ts @@ -17,7 +17,8 @@ export interface LegacyMigrationSource { accounts: ReadonlyArray ) => Promise readonly restore: ( - accounts: ReadonlyArray + accounts: ReadonlyArray, + options: { readonly preserveRecovery: boolean } ) => Promise readonly markActivating?: () => Promise } @@ -66,6 +67,7 @@ export async function migrateLegacyTenant(options: { let sourceBegan = false let destinationAttempted = false let activationAttempted = false + let rollbackConfirmed = false let sourceCompleted = false try { if (options.finalizeSource) { @@ -110,6 +112,7 @@ export async function migrateLegacyTenant(options: { allowLoopback: options.allowLoopback, fetch: options.fetch, }) + rollbackConfirmed = true } catch { if (activationAttempted) { throw new Error("hosted migration failed and destination rollback failed") @@ -118,7 +121,9 @@ export async function migrateLegacyTenant(options: { } if (sourceBegan && !sourceCompleted) { try { - await options.source.restore(accounts) + await options.source.restore(accounts, { + preserveRecovery: destinationAttempted && !rollbackConfirmed, + }) } catch { throw new Error("hosted migration failed and source restoration failed") } From c076e32489fc5c8f3de844b668a25e06f9db1037 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 3 Aug 2026 13:51:37 -0700 Subject: [PATCH 12/15] fix: model migration outcomes atomically --- .../packages/worker/src/legacy-migration.ts | 63 ++++++++++++++----- internal/accounts/codex_store.go | 36 ++++++++++- 2 files changed, 81 insertions(+), 18 deletions(-) diff --git a/cloudflare/packages/worker/src/legacy-migration.ts b/cloudflare/packages/worker/src/legacy-migration.ts index a90ade98a..aac2ff009 100644 --- a/cloudflare/packages/worker/src/legacy-migration.ts +++ b/cloudflare/packages/worker/src/legacy-migration.ts @@ -78,7 +78,6 @@ export async function migrateLegacyTenant(options: { } else { accounts = await options.source.list() } - destinationAttempted = true const migrated = await migrateLegacyAccountsToHosted({ destinationUrl, tenantKey, @@ -86,10 +85,12 @@ export async function migrateLegacyTenant(options: { accounts, allowLoopback: options.allowLoopback, fetch: options.fetch, + onRequestStart: () => { + destinationAttempted = true + }, }) if (sourceBegan) { await options.source.markActivating?.() - activationAttempted = true await activateLegacyMigrationDestination({ destinationUrl, tenantKey, @@ -97,6 +98,9 @@ export async function migrateLegacyTenant(options: { accountIds: accounts.map((account) => account.id), allowLoopback: options.allowLoopback, fetch: options.fetch, + onRequestStart: () => { + activationAttempted = true + }, }) await options.source.complete(accounts) sourceCompleted = true @@ -104,18 +108,22 @@ export async function migrateLegacyTenant(options: { return migrated } catch (error) { if (destinationAttempted && !sourceCompleted) { - try { - await rollbackLegacyMigrationDestination({ - destinationUrl, - tenantKey, - migrationId, - allowLoopback: options.allowLoopback, - fetch: options.fetch, - }) + if (isAuthoritativeStageRejection(error)) { rollbackConfirmed = true - } catch { - if (activationAttempted) { - throw new Error("hosted migration failed and destination rollback failed") + } else { + try { + await rollbackLegacyMigrationDestination({ + destinationUrl, + tenantKey, + migrationId, + allowLoopback: options.allowLoopback, + fetch: options.fetch, + }) + rollbackConfirmed = true + } catch { + if (activationAttempted) { + throw new Error("hosted migration failed and destination rollback failed") + } } } } @@ -139,6 +147,7 @@ export async function migrateLegacyAccountsToHosted(options: { readonly accounts: ReadonlyArray readonly allowLoopback?: boolean readonly fetch?: typeof fetch + readonly onRequestStart?: () => void }): Promise { const destination = migrationDestination( options.destinationUrl, @@ -161,6 +170,7 @@ export async function migrateLegacyAccountsToHosted(options: { body: { migrationId, accounts: uploads }, fetch: options.fetch, operation: "stage", + onRequestStart: options.onRequestStart, }) if ( body["ok"] !== true || @@ -182,6 +192,7 @@ async function activateLegacyMigrationDestination(options: { readonly accountIds: ReadonlyArray readonly allowLoopback?: boolean readonly fetch?: typeof fetch + readonly onRequestStart?: () => void }): Promise { const body = await migrationDestinationRequest({ destination: migrationDestination( @@ -196,6 +207,7 @@ async function activateLegacyMigrationDestination(options: { }, fetch: options.fetch, operation: "activation", + onRequestStart: options.onRequestStart, }) if (body["ok"] !== true) { throw new Error("hosted migration returned an invalid activation") @@ -232,9 +244,11 @@ async function migrationDestinationRequest(options: { readonly body: Record readonly fetch?: typeof fetch readonly operation: string + readonly onRequestStart?: () => void }): Promise> { let response: Response try { + options.onRequestStart?.() response = await (options.fetch ?? fetch)( `${options.destination}${options.path}`, { @@ -252,8 +266,9 @@ async function migrationDestinationRequest(options: { throw new Error("hosted migration destination is unavailable") } if (!response.ok) { - throw new Error( - `hosted migration ${options.operation} failed (${response.status})` + throw new MigrationDestinationResponseError( + options.operation, + response.status ) } const body = await response.json().catch(() => null) @@ -263,6 +278,24 @@ async function migrationDestinationRequest(options: { return body as Record } +class MigrationDestinationResponseError extends Error { + constructor( + readonly operation: string, + readonly status: number + ) { + super(`hosted migration ${operation} failed (${status})`) + this.name = "MigrationDestinationResponseError" + } +} + +function isAuthoritativeStageRejection(error: unknown): boolean { + return ( + error instanceof MigrationDestinationResponseError && + error.operation === "stage" && + [400, 401, 403, 404, 405, 413].includes(error.status) + ) +} + function normalizedMigrationID(value: string | undefined): string { const migrationId = value ?? "legacy-migration" if (!migrationIdPattern.test(migrationId)) { diff --git a/internal/accounts/codex_store.go b/internal/accounts/codex_store.go index a93d46dd2..a7627d657 100644 --- a/internal/accounts/codex_store.go +++ b/internal/accounts/codex_store.go @@ -133,6 +133,14 @@ func (s CodexStore) ListStored() ([]StoredCodexAccount, error) { } func (s CodexStore) listStored(includeInactiveMigrations bool) ([]StoredCodexAccount, error) { + var activeMigrationBatches map[string]struct{} + if !includeInactiveMigrations { + var err error + activeMigrationBatches, err = s.activeMigrationBatchSnapshot() + if err != nil { + return nil, err + } + } files, err := os.ReadDir(s.Dir) if errors.Is(err, os.ErrNotExist) { return nil, nil @@ -158,11 +166,10 @@ func (s CodexStore) listStored(includeInactiveMigrations bool) ([]StoredCodexAcc return nil, fmt.Errorf("parse %s: %w", path, err) } if !includeInactiveMigrations && stored.MigrationBatchID != "" { - active, err := s.migrationBatchActive(stored.MigrationBatchID) - if err != nil { + if err := validateMigrationBatchID(stored.MigrationBatchID); err != nil { return nil, err } - if !active { + if _, active := activeMigrationBatches[stored.MigrationBatchID]; !active { continue } } @@ -600,6 +607,29 @@ func (s CodexStore) migrationBatchMarker(batchID string) string { return filepath.Join(s.Dir, ".migration-batches", batchID+".active.json") } +func (s CodexStore) activeMigrationBatchSnapshot() (map[string]struct{}, error) { + active := make(map[string]struct{}) + entries, err := os.ReadDir(filepath.Join(s.Dir, ".migration-batches")) + if errors.Is(err, os.ErrNotExist) { + return active, nil + } + if err != nil { + return nil, err + } + const suffix = ".active.json" + for _, entry := range entries { + if entry.IsDir() || !strings.HasSuffix(entry.Name(), suffix) { + continue + } + batchID := strings.TrimSuffix(entry.Name(), suffix) + if err := validateMigrationBatchID(batchID); err != nil { + continue + } + active[batchID] = struct{}{} + } + return active, nil +} + func (s CodexStore) migrationBatchActive(batchID string) (bool, error) { if batchID == "" { return false, nil From 39f440896ccb4f4ef5712d693e7faae89c36d50e Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:08:20 -0700 Subject: [PATCH 13/15] fix: preserve migration ownership through rollback --- internal/accounts/codex_auth.go | 6 +-- internal/accounts/codex_store.go | 64 ++++++++++++++++++++------------ 2 files changed, 44 insertions(+), 26 deletions(-) diff --git a/internal/accounts/codex_auth.go b/internal/accounts/codex_auth.go index f4d684ebd..49b9bb176 100644 --- a/internal/accounts/codex_auth.go +++ b/internal/accounts/codex_auth.go @@ -112,7 +112,7 @@ func (s CodexStore) SyncActiveToStore() error { return err } defer lock.Close() - account, found, err := s.FindStored(email) + account, found, err := s.findStoredExact(email) if err != nil || !found { return err } @@ -218,7 +218,7 @@ func (s CodexStore) refreshStored(ctx context.Context, client *http.Client, acco } defer lock.Close() - latest, found, err := s.FindStored(account.Email) + latest, found, err := s.findStoredExact(account.Email) if err != nil { logCodexRefreshFailed(ctx, s, account, force, err) return account, false, err @@ -277,7 +277,7 @@ func (s CodexStore) refreshStored(ctx context.Context, client *http.Client, acco } func (s CodexStore) recoverRefreshedAccount(previous StoredCodexAccount) (StoredCodexAccount, bool) { - latest, found, err := s.FindStored(previous.Email) + latest, found, err := s.findStoredExact(previous.Email) if err != nil || !found || latest.Auth.Tokens == nil { return StoredCodexAccount{}, false } diff --git a/internal/accounts/codex_store.go b/internal/accounts/codex_store.go index a7627d657..cd41d063a 100644 --- a/internal/accounts/codex_store.go +++ b/internal/accounts/codex_store.go @@ -129,6 +129,11 @@ func (s CodexStore) List() ([]Account, error) { } func (s CodexStore) ListStored() ([]StoredCodexAccount, error) { + lock, err := s.lockStoredAccount(migrationBatchControlLockID) + if err != nil { + return nil, err + } + defer lock.Close() return s.listStored(false) } @@ -306,6 +311,7 @@ func (s CodexStore) saveStoredUnlocked(account StoredCodexAccount) error { if existing.MigrationBatchID == "" || !active { return fmt.Errorf("account %q belongs to a different migration state", account.Email) } + account.MigrationBatchID = existing.MigrationBatchID } } else if !errors.Is(err, os.ErrNotExist) { return err @@ -340,30 +346,12 @@ func (s CodexStore) FindStored(identifier string) (StoredCodexAccount, bool, err if needle == "" { return StoredCodexAccount{}, false, nil } - directPath := filepath.Join(s.Dir, emailToFilename(needle)) - if body, err := os.ReadFile(directPath); err == nil { - var account StoredCodexAccount - if err := json.Unmarshal(body, &account); err != nil { - return StoredCodexAccount{}, false, err - } - if strings.EqualFold(strings.TrimSpace(account.Email), needle) { - if account.MigrationBatchID != "" { - active, activeErr := s.migrationBatchActive(account.MigrationBatchID) - if activeErr != nil { - return StoredCodexAccount{}, false, activeErr - } - if !active { - return StoredCodexAccount{}, false, nil - } - } - return account, true, nil - } - } else if !errors.Is(err, os.ErrNotExist) { - return StoredCodexAccount{}, false, err + if account, ok, err := s.findStoredExact(needle); err != nil || ok { + return account, ok, err } if !strings.HasPrefix(needle, "apikey:") && !strings.Contains(needle, "@") { - if account, ok, err := s.FindStored("apikey:" + needle); err != nil || ok { + if account, ok, err := s.findStoredExact("apikey:" + needle); err != nil || ok { return account, ok, err } } @@ -392,6 +380,36 @@ func (s CodexStore) FindStored(identifier string) (StoredCodexAccount, bool, err return matches[0], true, nil } +func (s CodexStore) findStoredExact(identifier string) (StoredCodexAccount, bool, error) { + needle := strings.TrimSpace(identifier) + if needle == "" { + return StoredCodexAccount{}, false, nil + } + directPath := filepath.Join(s.Dir, emailToFilename(needle)) + if body, err := os.ReadFile(directPath); err == nil { + var account StoredCodexAccount + if err := json.Unmarshal(body, &account); err != nil { + return StoredCodexAccount{}, false, err + } + if strings.EqualFold(strings.TrimSpace(account.Email), needle) { + if account.MigrationBatchID != "" { + active, activeErr := s.migrationBatchActive(account.MigrationBatchID) + if activeErr != nil { + return StoredCodexAccount{}, false, activeErr + } + if !active { + return StoredCodexAccount{}, false, nil + } + } + return account, true, nil + } + } else if !errors.Is(err, os.ErrNotExist) { + return StoredCodexAccount{}, false, err + } + + return StoredCodexAccount{}, false, nil +} + // StageMigrationBatch persists credentials outside every routing and refresh // path. A single activation marker later makes the complete batch visible. func (s CodexStore) StageMigrationBatch(batchID string, staged []StoredCodexAccount) error { @@ -657,7 +675,7 @@ func (s CodexStore) RemoveStored(identifier string) (StoredCodexAccount, bool, e return account, false, err } defer lock.Close() - account, ok, err = s.FindStored(identifier) + account, ok, err = s.findStoredExact(account.Email) if err != nil || !ok { return account, ok, err } @@ -686,7 +704,7 @@ func (s CodexStore) MigrateStoredAway(identifier string) (string, bool, error) { return "", false, err } defer lock.Close() - account, ok, err = s.FindStored(identifier) + account, ok, err = s.findStoredExact(account.Email) if err != nil || !ok { return "", ok, err } From 6a84429948aee481c04f195e72fafc787554a5c4 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:19:33 -0700 Subject: [PATCH 14/15] fix: serialize hosted migration attempts --- cloudflare/packages/worker/src/index.ts | 36 ++++++++++++------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/cloudflare/packages/worker/src/index.ts b/cloudflare/packages/worker/src/index.ts index 05e77e5ec..6ca348eed 100644 --- a/cloudflare/packages/worker/src/index.ts +++ b/cloudflare/packages/worker/src/index.ts @@ -2598,29 +2598,29 @@ export class SubrouterDurableObject extends DurableObject { }, }), }) - if (!input.finalizeSource) return await migrate() - const receipt = this.hostedMigrationState(orgId) - if (receipt?.phase === "completed") { - this.assertHostedMigrationBinding(receipt, binding) - const sourceAccounts = this.ctx.storage.sql - .exec( - "SELECT COUNT(*) AS count FROM accounts WHERE org_id = ?", - orgId - ) - .one().count - if (sourceAccounts !== 0) { - throw new Error("legacy source changed after hosted migration completed") - } - return { migrated: receipt.account_count } - } if (this.migrationPreparing) { throw new Error("hosted migration is already in progress") } - - // Stop new refreshes before waiting for the ones that already own a - // single-use refresh token. No source row changes until they settle. this.migrationPreparing = true try { + if (!input.finalizeSource) return await migrate() + const receipt = this.hostedMigrationState(orgId) + if (receipt?.phase === "completed") { + this.assertHostedMigrationBinding(receipt, binding) + const sourceAccounts = this.ctx.storage.sql + .exec( + "SELECT COUNT(*) AS count FROM accounts WHERE org_id = ?", + orgId + ) + .one().count + if (sourceAccounts !== 0) { + throw new Error("legacy source changed after hosted migration completed") + } + return { migrated: receipt.account_count } + } + + // Stop new refreshes before waiting for the ones that already own a + // single-use refresh token. No source row changes until they settle. const pending = this.hostedMigrationState(orgId) if (pending) { this.assertHostedMigrationBinding(pending, binding) From 487e0007ba7a1180f25c8a7d4f51c2bbc62b116f Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 3 Aug 2026 15:00:22 -0700 Subject: [PATCH 15/15] fix: align hosted migration text contract --- .../packages/worker/src/legacy-migration.ts | 38 +++++++++++++++---- internal/proxy/multitenant.go | 16 +++++--- 2 files changed, 41 insertions(+), 13 deletions(-) diff --git a/cloudflare/packages/worker/src/legacy-migration.ts b/cloudflare/packages/worker/src/legacy-migration.ts index aac2ff009..a1e1f1793 100644 --- a/cloudflare/packages/worker/src/legacy-migration.ts +++ b/cloudflare/packages/worker/src/legacy-migration.ts @@ -71,6 +71,7 @@ export async function migrateLegacyTenant(options: { let sourceCompleted = false try { if (options.finalizeSource) { + validatedMigrationUploads(await options.source.list()) // Mark the attempt before calling begin so a source with persisted // recovery state can undo a failure that happens after quiescing. sourceBegan = true @@ -155,13 +156,9 @@ export async function migrateLegacyAccountsToHosted(options: { ) const tenantKey = migrationTenantKey(options.tenantKey) const migrationId = normalizedMigrationID(options.migrationId) - if (options.accounts.length > maxMigrationAccounts) { - throw new Error("hosted migration account limit exceeded") - } - // Validate every source row before the first external mutation. A retry can // safely overwrite the same destination ids after a network interruption. - const uploads = options.accounts.map(migrationUpload) + const uploads = validatedMigrationUploads(options.accounts) const accountIds = uploads.map((upload) => String(upload.accountId)) const body = await migrationDestinationRequest({ destination, @@ -345,8 +342,8 @@ function migrationDestination(value: unknown, allowLoopback: boolean): string { } function migrationUpload(account: LegacyMigrationAccount): Record { - const id = normalizedString(account.id, 320) - const label = normalizedString(account.label, 320) + const id = normalizedMigrationText(account.id, 320) + const label = normalizedMigrationText(account.label, 320) if (!id || !label || !account.enabled || account.hasTotp) { throw new Error("legacy account cannot be migrated safely") } @@ -386,6 +383,33 @@ function migrationUpload(account: LegacyMigrationAccount): Record maxBytes + ) { + return null + } + return normalized +} + +function validatedMigrationUploads( + accounts: ReadonlyArray +): ReadonlyArray> { + if (accounts.length === 0 || accounts.length > maxMigrationAccounts) { + throw new Error("hosted migration account limit exceeded") + } + return accounts.map(migrationUpload) +} + function normalizedString(value: unknown, maxLength: number): string | null { if (typeof value !== "string") return null const normalized = value.trim() diff --git a/internal/proxy/multitenant.go b/internal/proxy/multitenant.go index 712f92355..e4dd4aba3 100644 --- a/internal/proxy/multitenant.go +++ b/internal/proxy/multitenant.go @@ -805,10 +805,10 @@ func storedTenantMigrationAccount(input tenantAccountUpload) (accounts.StoredCod input.Provider = strings.ToLower(strings.TrimSpace(input.Provider)) input.AccountID = strings.TrimSpace(input.AccountID) input.Label = strings.TrimSpace(input.Label) - if input.Label == "" || len(input.Label) > 320 { - return accounts.StoredCodexAccount{}, errors.New("account label is required") + if !validTenantAccountText(input.Label) { + return accounts.StoredCodexAccount{}, errors.New("account label is invalid") } - if input.AccountID == "" || len(input.AccountID) > 320 || containsTerminalControl(input.AccountID) { + if !validTenantAccountText(input.AccountID) { return accounts.StoredCodexAccount{}, errors.New("account id is invalid") } switch input.Provider { @@ -867,12 +867,12 @@ func handleTenantAccountUpload(server *Server, w http.ResponseWriter, r *http.Re input.AccountID = strings.TrimSpace(input.AccountID) input.Label = strings.TrimSpace(input.Label) input.TargetAccountID = strings.TrimSpace(input.TargetAccountID) - if input.Label == "" || len(input.Label) > 320 { - http.Error(w, "account label is required", http.StatusBadRequest) + if !validTenantAccountText(input.Label) { + http.Error(w, "account label is invalid", http.StatusBadRequest) return } if input.AccountID != "" && - (len(input.AccountID) > 320 || containsTerminalControl(input.AccountID)) { + !validTenantAccountText(input.AccountID) { http.Error(w, "account id is invalid", http.StatusBadRequest) return } @@ -962,6 +962,10 @@ func handleTenantAccountUpload(server *Server, w http.ResponseWriter, r *http.Re }}) } +func validTenantAccountText(value string) bool { + return value != "" && len(value) <= 320 && !containsTerminalControl(value) +} + func handleTenantAccountDelete(server *Server, w http.ResponseWriter, r *http.Request) { if server.AccountRef == nil { http.Error(w, "tenant account store unavailable", http.StatusServiceUnavailable)