From c71f45703a9ca0fc3854704bd722eb2369ee5004 Mon Sep 17 00:00:00 2001 From: Flo Date: Mon, 6 Oct 2025 10:54:21 +0200 Subject: [PATCH 1/7] fix: identity concurrency creation err --- .../api/routes/v2_keys_create_key/200_test.go | 177 ++++++++++++++++++ .../api/routes/v2_keys_create_key/handler.go | 29 ++- .../api/routes/v2_keys_update_key/handler.go | 2 +- 3 files changed, 202 insertions(+), 6 deletions(-) diff --git a/go/apps/api/routes/v2_keys_create_key/200_test.go b/go/apps/api/routes/v2_keys_create_key/200_test.go index 018e4eedf8..4f594559de 100644 --- a/go/apps/api/routes/v2_keys_create_key/200_test.go +++ b/go/apps/api/routes/v2_keys_create_key/200_test.go @@ -189,3 +189,180 @@ func TestCreateKeyWithEncryption(t *testing.T) { require.Equal(t, keyEncryption.KeyID, res.Body.Data.KeyId) require.Equal(t, keyEncryption.WorkspaceID, h.Resources().UserWorkspace.ID) } + +func TestCreateKeyWithDuplicateExternalId(t *testing.T) { + t.Parallel() + + h := testutil.NewHarness(t) + ctx := context.Background() + + route := &handler.Handler{ + Logger: h.Logger, + DB: h.DB, + Keys: h.Keys, + Auditlogs: h.Auditlogs, + Vault: h.Vault, + } + + h.Register(route) + + // Create API using testutil helper + api := h.CreateApi(seed.CreateApiRequest{ + WorkspaceID: h.Resources().UserWorkspace.ID, + }) + + rootKey := h.CreateRootKey(h.Resources().UserWorkspace.ID, "api.*.create_key") + + headers := http.Header{ + "Content-Type": {"application/json"}, + "Authorization": {fmt.Sprintf("Bearer %s", rootKey)}, + } + + // Create first key with an externalId + externalID := "user_duplicate_test" + req1 := handler.Request{ + ApiId: api.ID, + ExternalId: &externalID, + } + + res1 := testutil.CallRoute[handler.Request, handler.Response](h, route, headers, req1) + require.Equal(t, 200, res1.Status) + require.NotNil(t, res1.Body) + require.NotEmpty(t, res1.Body.Data.KeyId) + + // Verify first key was created in database + key1, err := db.Query.FindKeyByID(ctx, h.DB.RO(), res1.Body.Data.KeyId) + require.NoError(t, err) + require.True(t, key1.IdentityID.Valid) + identityID1 := key1.IdentityID.String + + // Create second key with the same externalId + // This should trigger the duplicate identity handling + req2 := handler.Request{ + ApiId: api.ID, + ExternalId: &externalID, + } + + res2 := testutil.CallRoute[handler.Request, handler.Response](h, route, headers, req2) + require.Equal(t, 200, res2.Status, "Second key creation should succeed despite duplicate externalId") + require.NotNil(t, res2.Body) + require.NotEmpty(t, res2.Body.Data.KeyId) + + // Verify second key was created in database + key2, err := db.Query.FindKeyByID(ctx, h.DB.RO(), res2.Body.Data.KeyId) + require.NoError(t, err) + require.True(t, key2.IdentityID.Valid) + + // Both keys should reference the same identity + require.Equal(t, identityID1, key2.IdentityID.String, "Both keys should share the same identity ID") + + // Verify the identity exists + identity, err := db.Query.FindIdentity(ctx, h.DB.RO(), db.FindIdentityParams{ + WorkspaceID: h.Resources().UserWorkspace.ID, + Identity: externalID, + Deleted: false, + }) + require.NoError(t, err) + require.Equal(t, identityID1, identity.ID) + require.Equal(t, externalID, identity.ExternalID) +} + +func TestCreateKeyConcurrentWithSameExternalId(t *testing.T) { + t.Parallel() + + h := testutil.NewHarness(t) + ctx := context.Background() + + route := &handler.Handler{ + Logger: h.Logger, + DB: h.DB, + Keys: h.Keys, + Auditlogs: h.Auditlogs, + Vault: h.Vault, + } + + h.Register(route) + + // Create API using testutil helper + api := h.CreateApi(seed.CreateApiRequest{ + WorkspaceID: h.Resources().UserWorkspace.ID, + }) + + rootKey := h.CreateRootKey(h.Resources().UserWorkspace.ID, "api.*.create_key") + + headers := http.Header{ + "Content-Type": {"application/json"}, + "Authorization": {fmt.Sprintf("Bearer %s", rootKey)}, + } + + // Use same externalId for concurrent requests + externalID := "user_concurrent_test" + + // Create multiple keys concurrently with the same externalId + // This simulates the race condition where: + // 1. Request A checks if identity exists - doesn't find it + // 2. Request B checks if identity exists - doesn't find it + // 3. Request A tries to insert identity - succeeds + // 4. Request B tries to insert identity - gets duplicate key error + // 5. Request B handles the error by finding the existing identity + numConcurrent := 5 + results := make(chan testutil.TestResponse[handler.Response], numConcurrent) + errors := make(chan error, numConcurrent) + + for i := 0; i < numConcurrent; i++ { + go func() { + req := handler.Request{ + ApiId: api.ID, + ExternalId: &externalID, + } + res := testutil.CallRoute[handler.Request, handler.Response](h, route, headers, req) + if res.Status != 200 { + errors <- fmt.Errorf("unexpected status code: %d", res.Status) + return + } + results <- res + }() + } + + // Collect all results + keyIDs := make([]string, 0, numConcurrent) + for i := 0; i < numConcurrent; i++ { + select { + case res := <-results: + require.Equal(t, 200, res.Status) + require.NotNil(t, res.Body) + require.NotEmpty(t, res.Body.Data.KeyId) + keyIDs = append(keyIDs, res.Body.Data.KeyId) + case err := <-errors: + t.Fatal(err) + } + } + + // Verify all keys were created + require.Len(t, keyIDs, numConcurrent) + + // Verify all keys reference the same identity + var sharedIdentityID string + for i, keyID := range keyIDs { + key, err := db.Query.FindKeyByID(ctx, h.DB.RO(), keyID) + require.NoError(t, err) + require.True(t, key.IdentityID.Valid) + + if i == 0 { + sharedIdentityID = key.IdentityID.String + } else { + require.Equal(t, sharedIdentityID, key.IdentityID.String, + "All concurrent keys should reference the same identity") + } + } + + // Verify only one identity was created + identity, err := db.Query.FindIdentity(ctx, h.DB.RO(), db.FindIdentityParams{ + WorkspaceID: h.Resources().UserWorkspace.ID, + Identity: externalID, + Deleted: false, + }) + require.NoError(t, err) + require.Equal(t, sharedIdentityID, identity.ID) + require.Equal(t, externalID, identity.ExternalID) +} diff --git a/go/apps/api/routes/v2_keys_create_key/handler.go b/go/apps/api/routes/v2_keys_create_key/handler.go index 489e15a731..4d3918c3d0 100644 --- a/go/apps/api/routes/v2_keys_create_key/handler.go +++ b/go/apps/api/routes/v2_keys_create_key/handler.go @@ -233,12 +233,31 @@ func (h *Handler) Handle(ctx context.Context, s *zen.Session) error { Meta: []byte("{}"), }) if err != nil { - return fault.Wrap(err, - fault.Code(codes.App.Internal.ServiceUnavailable.URN()), - fault.Internal("failed to create identity"), - fault.Public("Failed to create identity."), - ) + // Incase of duplicate key error just find existing identity + if !db.IsDuplicateKeyError(err) { + return fault.Wrap(err, + fault.Code(codes.App.Internal.ServiceUnavailable.URN()), + fault.Internal("failed to create identity"), + fault.Public("Failed to create identity."), + ) + } + + identity, err = db.Query.FindIdentity(ctx, h.DB.RO(), db.FindIdentityParams{ + WorkspaceID: auth.AuthorizedWorkspaceID, + Identity: externalID, + Deleted: false, + }) + if err != nil { + return fault.Wrap(err, + fault.Code(codes.App.Internal.ServiceUnavailable.URN()), + fault.Internal("failed to find identity"), + fault.Public("Failed to find identity."), + ) + } + + identityID = identity.ID } + insertKeyParams.IdentityID = sql.NullString{Valid: true, String: identityID} } else { // Use existing identity diff --git a/go/apps/api/routes/v2_keys_update_key/handler.go b/go/apps/api/routes/v2_keys_update_key/handler.go index 0bc82387fb..cd6ba70c75 100644 --- a/go/apps/api/routes/v2_keys_update_key/handler.go +++ b/go/apps/api/routes/v2_keys_update_key/handler.go @@ -183,7 +183,7 @@ func (h *Handler) Handle(ctx context.Context, s *zen.Session) error { ) } - identity, err = db.Query.FindIdentity(ctx, tx, db.FindIdentityParams{ + identity, err = db.Query.FindIdentity(ctx, h.DB.RO(), db.FindIdentityParams{ WorkspaceID: auth.AuthorizedWorkspaceID, Identity: externalID, Deleted: false, From 60ea1e6a7cad54a20c9ac0607a3d0aa5da1e3d4c Mon Sep 17 00:00:00 2001 From: Flo Date: Mon, 6 Oct 2025 10:55:34 +0200 Subject: [PATCH 2/7] remove other test --- .../api/routes/v2_keys_create_key/200_test.go | 77 ------------------- 1 file changed, 77 deletions(-) diff --git a/go/apps/api/routes/v2_keys_create_key/200_test.go b/go/apps/api/routes/v2_keys_create_key/200_test.go index 4f594559de..a1b7cac053 100644 --- a/go/apps/api/routes/v2_keys_create_key/200_test.go +++ b/go/apps/api/routes/v2_keys_create_key/200_test.go @@ -190,83 +190,6 @@ func TestCreateKeyWithEncryption(t *testing.T) { require.Equal(t, keyEncryption.WorkspaceID, h.Resources().UserWorkspace.ID) } -func TestCreateKeyWithDuplicateExternalId(t *testing.T) { - t.Parallel() - - h := testutil.NewHarness(t) - ctx := context.Background() - - route := &handler.Handler{ - Logger: h.Logger, - DB: h.DB, - Keys: h.Keys, - Auditlogs: h.Auditlogs, - Vault: h.Vault, - } - - h.Register(route) - - // Create API using testutil helper - api := h.CreateApi(seed.CreateApiRequest{ - WorkspaceID: h.Resources().UserWorkspace.ID, - }) - - rootKey := h.CreateRootKey(h.Resources().UserWorkspace.ID, "api.*.create_key") - - headers := http.Header{ - "Content-Type": {"application/json"}, - "Authorization": {fmt.Sprintf("Bearer %s", rootKey)}, - } - - // Create first key with an externalId - externalID := "user_duplicate_test" - req1 := handler.Request{ - ApiId: api.ID, - ExternalId: &externalID, - } - - res1 := testutil.CallRoute[handler.Request, handler.Response](h, route, headers, req1) - require.Equal(t, 200, res1.Status) - require.NotNil(t, res1.Body) - require.NotEmpty(t, res1.Body.Data.KeyId) - - // Verify first key was created in database - key1, err := db.Query.FindKeyByID(ctx, h.DB.RO(), res1.Body.Data.KeyId) - require.NoError(t, err) - require.True(t, key1.IdentityID.Valid) - identityID1 := key1.IdentityID.String - - // Create second key with the same externalId - // This should trigger the duplicate identity handling - req2 := handler.Request{ - ApiId: api.ID, - ExternalId: &externalID, - } - - res2 := testutil.CallRoute[handler.Request, handler.Response](h, route, headers, req2) - require.Equal(t, 200, res2.Status, "Second key creation should succeed despite duplicate externalId") - require.NotNil(t, res2.Body) - require.NotEmpty(t, res2.Body.Data.KeyId) - - // Verify second key was created in database - key2, err := db.Query.FindKeyByID(ctx, h.DB.RO(), res2.Body.Data.KeyId) - require.NoError(t, err) - require.True(t, key2.IdentityID.Valid) - - // Both keys should reference the same identity - require.Equal(t, identityID1, key2.IdentityID.String, "Both keys should share the same identity ID") - - // Verify the identity exists - identity, err := db.Query.FindIdentity(ctx, h.DB.RO(), db.FindIdentityParams{ - WorkspaceID: h.Resources().UserWorkspace.ID, - Identity: externalID, - Deleted: false, - }) - require.NoError(t, err) - require.Equal(t, identityID1, identity.ID) - require.Equal(t, externalID, identity.ExternalID) -} - func TestCreateKeyConcurrentWithSameExternalId(t *testing.T) { t.Parallel() From bf8a794aabaad331c1a11640f3caededf7c943f1 Mon Sep 17 00:00:00 2001 From: Flo Date: Mon, 6 Oct 2025 15:55:47 +0200 Subject: [PATCH 3/7] fix: just retry whole TX --- .../api/routes/v2_keys_create_key/200_test.go | 2 +- .../api/routes/v2_keys_create_key/handler.go | 697 +++++++-------- .../api/routes/v2_keys_update_key/handler.go | 816 +++++++++--------- go/pkg/db/handle_err_deadlock.go | 16 + 4 files changed, 785 insertions(+), 746 deletions(-) create mode 100644 go/pkg/db/handle_err_deadlock.go diff --git a/go/apps/api/routes/v2_keys_create_key/200_test.go b/go/apps/api/routes/v2_keys_create_key/200_test.go index a1b7cac053..7a51d49668 100644 --- a/go/apps/api/routes/v2_keys_create_key/200_test.go +++ b/go/apps/api/routes/v2_keys_create_key/200_test.go @@ -232,7 +232,7 @@ func TestCreateKeyConcurrentWithSameExternalId(t *testing.T) { results := make(chan testutil.TestResponse[handler.Response], numConcurrent) errors := make(chan error, numConcurrent) - for i := 0; i < numConcurrent; i++ { + for range numConcurrent { go func() { req := handler.Request{ ApiId: api.ID, diff --git a/go/apps/api/routes/v2_keys_create_key/handler.go b/go/apps/api/routes/v2_keys_create_key/handler.go index 4d3918c3d0..a8b211b9f1 100644 --- a/go/apps/api/routes/v2_keys_create_key/handler.go +++ b/go/apps/api/routes/v2_keys_create_key/handler.go @@ -178,434 +178,445 @@ func (h *Handler) Handle(ctx context.Context, s *zen.Session) error { now := time.Now().UnixMilli() - err = db.Tx(ctx, h.DB.RW(), func(ctx context.Context, tx db.DBTX) error { - insertKeyParams := db.InsertKeyParams{ - ID: keyID, - KeyringID: api.KeyAuthID.String, - Hash: keyResult.Hash, - Start: keyResult.Start, - WorkspaceID: auth.AuthorizedWorkspaceID, - ForWorkspaceID: sql.NullString{String: "", Valid: false}, - CreatedAtM: now, - Enabled: true, - RemainingRequests: sql.NullInt32{Int32: 0, Valid: false}, - RefillDay: sql.NullInt16{Int16: 0, Valid: false}, - RefillAmount: sql.NullInt32{Int32: 0, Valid: false}, - Name: sql.NullString{String: "", Valid: false}, - IdentityID: sql.NullString{String: "", Valid: false}, - Meta: sql.NullString{String: "", Valid: false}, - Expires: sql.NullTime{Time: time.Time{}, Valid: false}, - } - - // Set optional fields - if req.Name != nil { - insertKeyParams.Name = sql.NullString{String: *req.Name, Valid: true} - } + // Retry transaction up to 3 times on deadlock or identity creation race + var txErr error + for attempt := 0; attempt < 3; attempt++ { + txErr = db.Tx(ctx, h.DB.RW(), func(ctx context.Context, tx db.DBTX) error { + insertKeyParams := db.InsertKeyParams{ + ID: keyID, + KeyringID: api.KeyAuthID.String, + Hash: keyResult.Hash, + Start: keyResult.Start, + WorkspaceID: auth.AuthorizedWorkspaceID, + ForWorkspaceID: sql.NullString{String: "", Valid: false}, + CreatedAtM: now, + Enabled: true, + RemainingRequests: sql.NullInt32{Int32: 0, Valid: false}, + RefillDay: sql.NullInt16{Int16: 0, Valid: false}, + RefillAmount: sql.NullInt32{Int32: 0, Valid: false}, + Name: sql.NullString{String: "", Valid: false}, + IdentityID: sql.NullString{String: "", Valid: false}, + Meta: sql.NullString{String: "", Valid: false}, + Expires: sql.NullTime{Time: time.Time{}, Valid: false}, + } - // Handle identity creation/lookup from externalId - if req.ExternalId != nil { - externalID := *req.ExternalId + // Set optional fields + if req.Name != nil { + insertKeyParams.Name = sql.NullString{String: *req.Name, Valid: true} + } - // Try to find existing identity - identity, err := db.Query.FindIdentity(ctx, tx, db.FindIdentityParams{ - WorkspaceID: auth.AuthorizedWorkspaceID, - Identity: externalID, - Deleted: false, - }) + // Handle identity creation/lookup from externalId + if req.ExternalId != nil { + externalID := *req.ExternalId - if err != nil { - if !db.IsNotFound(err) { - return fault.Wrap(err, - fault.Code(codes.App.Internal.ServiceUnavailable.URN()), - fault.Internal("failed to find identity"), - fault.Public("Failed to find identity."), - ) - } - - // Create new identity - identityID := uid.New(uid.IdentityPrefix) - err = db.Query.InsertIdentity(ctx, tx, db.InsertIdentityParams{ - ID: identityID, - ExternalID: externalID, + // Try to find existing identity + identity, err := db.Query.FindIdentity(ctx, tx, db.FindIdentityParams{ WorkspaceID: auth.AuthorizedWorkspaceID, - Environment: "default", - CreatedAt: now, - Meta: []byte("{}"), + Identity: externalID, + Deleted: false, }) + if err != nil { - // Incase of duplicate key error just find existing identity - if !db.IsDuplicateKeyError(err) { + if !db.IsNotFound(err) { return fault.Wrap(err, fault.Code(codes.App.Internal.ServiceUnavailable.URN()), - fault.Internal("failed to create identity"), - fault.Public("Failed to create identity."), + fault.Internal("failed to find identity"), + fault.Public("Failed to find identity."), ) } - identity, err = db.Query.FindIdentity(ctx, h.DB.RO(), db.FindIdentityParams{ + // Create new identity + identityID := uid.New(uid.IdentityPrefix) + err = db.Query.InsertIdentity(ctx, tx, db.InsertIdentityParams{ + ID: identityID, + ExternalID: externalID, WorkspaceID: auth.AuthorizedWorkspaceID, - Identity: externalID, - Deleted: false, + Environment: "default", + CreatedAt: now, + Meta: []byte("{}"), }) if err != nil { + // Don't wrap duplicate key or deadlock errors - let retry loop handle them + if db.IsDuplicateKeyError(err) || db.IsDeadlockError(err) { + return err + } return fault.Wrap(err, fault.Code(codes.App.Internal.ServiceUnavailable.URN()), - fault.Internal("failed to find identity"), - fault.Public("Failed to find identity."), + fault.Internal("failed to create identity"), + fault.Public("Failed to create identity."), ) } - identityID = identity.ID + insertKeyParams.IdentityID = sql.NullString{Valid: true, String: identityID} + } else { + // Use existing identity + insertKeyParams.IdentityID = sql.NullString{Valid: true, String: identity.ID} } - - insertKeyParams.IdentityID = sql.NullString{Valid: true, String: identityID} - } else { - // Use existing identity - insertKeyParams.IdentityID = sql.NullString{Valid: true, String: identity.ID} } - } - - // Note: owner_id is set to null in the SQL query, so we skip setting it here - if req.Meta != nil { - metaBytes, marshalErr := json.Marshal(*req.Meta) - if marshalErr != nil { - return fault.Wrap(marshalErr, - fault.Code(codes.App.Validation.InvalidInput.URN()), - fault.Internal("failed to marshal meta"), fault.Public("Invalid metadata format."), - ) - } - - insertKeyParams.Meta = sql.NullString{String: string(metaBytes), Valid: true} - } - if req.Expires != nil { - insertKeyParams.Expires = sql.NullTime{Time: time.UnixMilli(*req.Expires), Valid: true} - } - - if req.Credits != nil { - if req.Credits.Remaining.IsSpecified() { - insertKeyParams.RemainingRequests = sql.NullInt32{ - Int32: int32(req.Credits.Remaining.MustGet()), // nolint:gosec - Valid: true, + // Note: owner_id is set to null in the SQL query, so we skip setting it here + if req.Meta != nil { + metaBytes, marshalErr := json.Marshal(*req.Meta) + if marshalErr != nil { + return fault.Wrap(marshalErr, + fault.Code(codes.App.Validation.InvalidInput.URN()), + fault.Internal("failed to marshal meta"), fault.Public("Invalid metadata format."), + ) } - } - if req.Credits.Refill != nil { - insertKeyParams.RefillAmount = sql.NullInt32{ - Int32: int32(req.Credits.Refill.Amount), // nolint:gosec - Valid: true, - } + insertKeyParams.Meta = sql.NullString{String: string(metaBytes), Valid: true} + } - if req.Credits.Refill.Interval == openapi.KeyCreditsRefillIntervalMonthly { - if req.Credits.Refill.RefillDay == nil { - return fault.New("missing refillDay", - fault.Code(codes.App.Validation.InvalidInput.URN()), - fault.Internal("refillDay required for monthly interval"), - fault.Public("`refillDay` must be provided when the refill interval is `monthly`."), - ) - } + if req.Expires != nil { + insertKeyParams.Expires = sql.NullTime{Time: time.UnixMilli(*req.Expires), Valid: true} + } - insertKeyParams.RefillDay = sql.NullInt16{ - Int16: int16(*req.Credits.Refill.RefillDay), // nolint:gosec + if req.Credits != nil { + if req.Credits.Remaining.IsSpecified() { + insertKeyParams.RemainingRequests = sql.NullInt32{ + Int32: int32(req.Credits.Remaining.MustGet()), // nolint:gosec Valid: true, } } - } - } - // Set enabled status (default true) - if req.Enabled != nil { - insertKeyParams.Enabled = *req.Enabled - } - - err = db.Query.InsertKey(ctx, tx, insertKeyParams) - if err != nil { - return fault.Wrap(err, - fault.Code(codes.App.Internal.ServiceUnavailable.URN()), - fault.Internal("database error"), fault.Public("Failed to create key."), - ) - } - - if encryption != nil { - err = db.Query.InsertKeyEncryption(ctx, tx, db.InsertKeyEncryptionParams{ - WorkspaceID: auth.AuthorizedWorkspaceID, - KeyID: keyID, - CreatedAt: now, - Encrypted: encryption.GetEncrypted(), - EncryptionKeyID: encryption.GetKeyId(), - }) - if err != nil { - return fault.Wrap(err, - fault.Code(codes.App.Internal.ServiceUnavailable.URN()), - fault.Internal("database error"), fault.Public("Failed to create key encryption."), - ) - } - } + if req.Credits.Refill != nil { + insertKeyParams.RefillAmount = sql.NullInt32{ + Int32: int32(req.Credits.Refill.Amount), // nolint:gosec + Valid: true, + } - if req.Ratelimits != nil && len(*req.Ratelimits) > 0 { - ratelimitsToInsert := make([]db.InsertKeyRatelimitParams, len(*req.Ratelimits)) - for i, ratelimit := range *req.Ratelimits { - ratelimitID := uid.New(uid.RatelimitPrefix) - ratelimitsToInsert[i] = db.InsertKeyRatelimitParams{ - ID: ratelimitID, - WorkspaceID: auth.AuthorizedWorkspaceID, - KeyID: sql.NullString{String: keyID, Valid: true}, - Name: ratelimit.Name, - Limit: int32(ratelimit.Limit), // nolint:gosec - Duration: ratelimit.Duration, - CreatedAt: now, - AutoApply: ratelimit.AutoApply, + if req.Credits.Refill.Interval == openapi.KeyCreditsRefillIntervalMonthly { + if req.Credits.Refill.RefillDay == nil { + return fault.New("missing refillDay", + fault.Code(codes.App.Validation.InvalidInput.URN()), + fault.Internal("refillDay required for monthly interval"), + fault.Public("`refillDay` must be provided when the refill interval is `monthly`."), + ) + } + + insertKeyParams.RefillDay = sql.NullInt16{ + Int16: int16(*req.Credits.Refill.RefillDay), // nolint:gosec + Valid: true, + } + } } } - err = db.BulkQuery.InsertKeyRatelimits(ctx, tx, ratelimitsToInsert) - if err != nil { - return fault.Wrap(err, - fault.Code(codes.App.Internal.ServiceUnavailable.URN()), - fault.Internal("database error"), fault.Public("Failed to create rate limit."), - ) + // Set enabled status (default true) + if req.Enabled != nil { + insertKeyParams.Enabled = *req.Enabled } - } - // 11. Handle permissions if provided - with auto-creation - var auditLogs []auditlog.AuditLog - if req.Permissions != nil { - existingPermissions, err := db.Query.FindPermissionsBySlugs(ctx, tx, db.FindPermissionsBySlugsParams{ - WorkspaceID: auth.AuthorizedWorkspaceID, - Slugs: *req.Permissions, - }) + err = db.Query.InsertKey(ctx, tx, insertKeyParams) if err != nil { return fault.Wrap(err, fault.Code(codes.App.Internal.ServiceUnavailable.URN()), - fault.Internal("database error"), - fault.Public("Failed to retrieve permissions."), + fault.Internal("database error"), fault.Public("Failed to create key."), ) } - existingPermMap := make(map[string]db.Permission) - for _, p := range existingPermissions { - existingPermMap[p.Slug] = p + if encryption != nil { + err = db.Query.InsertKeyEncryption(ctx, tx, db.InsertKeyEncryptionParams{ + WorkspaceID: auth.AuthorizedWorkspaceID, + KeyID: keyID, + CreatedAt: now, + Encrypted: encryption.GetEncrypted(), + EncryptionKeyID: encryption.GetKeyId(), + }) + if err != nil { + return fault.Wrap(err, + fault.Code(codes.App.Internal.ServiceUnavailable.URN()), + fault.Internal("database error"), fault.Public("Failed to create key encryption."), + ) + } } - permissionsToCreate := []db.InsertPermissionParams{} - requestedPermissions := []db.Permission{} - - for _, requestedSlug := range *req.Permissions { - existingPerm, exists := existingPermMap[requestedSlug] - if exists { - requestedPermissions = append(requestedPermissions, existingPerm) - continue + if req.Ratelimits != nil && len(*req.Ratelimits) > 0 { + ratelimitsToInsert := make([]db.InsertKeyRatelimitParams, len(*req.Ratelimits)) + for i, ratelimit := range *req.Ratelimits { + ratelimitID := uid.New(uid.RatelimitPrefix) + ratelimitsToInsert[i] = db.InsertKeyRatelimitParams{ + ID: ratelimitID, + WorkspaceID: auth.AuthorizedWorkspaceID, + KeyID: sql.NullString{String: keyID, Valid: true}, + Name: ratelimit.Name, + Limit: int32(ratelimit.Limit), // nolint:gosec + Duration: ratelimit.Duration, + CreatedAt: now, + AutoApply: ratelimit.AutoApply, + } } - newPermID := uid.New(uid.PermissionPrefix) - permissionsToCreate = append(permissionsToCreate, db.InsertPermissionParams{ - PermissionID: newPermID, - WorkspaceID: auth.AuthorizedWorkspaceID, - Name: requestedSlug, - Slug: requestedSlug, - Description: dbtype.NullString{String: "", Valid: false}, - CreatedAtM: now, - }) - - requestedPermissions = append(requestedPermissions, db.Permission{ - ID: newPermID, - Name: requestedSlug, - Slug: requestedSlug, - CreatedAtM: now, - WorkspaceID: auth.AuthorizedWorkspaceID, - Description: dbtype.NullString{String: "", Valid: false}, - UpdatedAtM: sql.NullInt64{Int64: 0, Valid: false}, - }) - } - - if len(permissionsToCreate) > 0 { - err = db.BulkQuery.InsertPermissions(ctx, tx, permissionsToCreate) + err = db.BulkQuery.InsertKeyRatelimits(ctx, tx, ratelimitsToInsert) if err != nil { return fault.Wrap(err, fault.Code(codes.App.Internal.ServiceUnavailable.URN()), - fault.Internal("database error"), - fault.Public("Failed to create permissions."), + fault.Internal("database error"), fault.Public("Failed to create rate limit."), ) } } - permissionsToInsert := []db.InsertKeyPermissionParams{} - for _, reqPerm := range requestedPermissions { - permissionsToInsert = append(permissionsToInsert, db.InsertKeyPermissionParams{ - KeyID: keyID, - PermissionID: reqPerm.ID, - WorkspaceID: auth.AuthorizedWorkspaceID, - CreatedAt: now, - UpdatedAt: sql.NullInt64{Valid: false, Int64: 0}, - }) - - auditLogs = append(auditLogs, auditlog.AuditLog{ + // 11. Handle permissions if provided - with auto-creation + var auditLogs []auditlog.AuditLog + if req.Permissions != nil { + existingPermissions, err := db.Query.FindPermissionsBySlugs(ctx, tx, db.FindPermissionsBySlugsParams{ WorkspaceID: auth.AuthorizedWorkspaceID, - Event: auditlog.AuthConnectPermissionKeyEvent, - ActorType: auditlog.RootKeyActor, - ActorID: auth.Key.ID, - ActorName: "root key", - ActorMeta: map[string]any{}, - Display: fmt.Sprintf("Added permission %s to key %s", reqPerm.Slug, keyID), - RemoteIP: s.Location(), - UserAgent: s.UserAgent(), - Resources: []auditlog.AuditLogResource{ - { - Type: auditlog.KeyResourceType, - ID: keyID, - Name: insertKeyParams.Name.String, - DisplayName: insertKeyParams.Name.String, - Meta: map[string]any{}, - }, - { - Type: auditlog.PermissionResourceType, - ID: reqPerm.ID, - Name: reqPerm.Slug, - DisplayName: reqPerm.Slug, - Meta: map[string]any{}, - }, - }, + Slugs: *req.Permissions, }) - } - - if len(permissionsToInsert) > 0 { - err = db.BulkQuery.InsertKeyPermissions(ctx, tx, permissionsToInsert) if err != nil { return fault.Wrap(err, fault.Code(codes.App.Internal.ServiceUnavailable.URN()), fault.Internal("database error"), - fault.Public("Failed to assign permissions."), + fault.Public("Failed to retrieve permissions."), ) } - } - } - // 12. Handle roles if provided - with auto-creation - if req.Roles != nil { - existingRoles, err := db.Query.FindRolesByNames(ctx, tx, db.FindRolesByNamesParams{ - WorkspaceID: auth.AuthorizedWorkspaceID, - Names: *req.Roles, - }) - if err != nil { - return fault.Wrap(err, - fault.Code(codes.App.Internal.ServiceUnavailable.URN()), - fault.Internal("database error"), - fault.Public("Failed to retrieve roles."), - ) - } + existingPermMap := make(map[string]db.Permission) + for _, p := range existingPermissions { + existingPermMap[p.Slug] = p + } - // Find which roles need to be created - existingRoleMap := make(map[string]db.FindRolesByNamesRow) - for _, r := range existingRoles { - existingRoleMap[r.Name] = r - } + permissionsToCreate := []db.InsertPermissionParams{} + requestedPermissions := []db.Permission{} - // Create missing roles in bulk and build final list - requestedRoles := []db.FindRolesByNamesRow{} + for _, requestedSlug := range *req.Permissions { + existingPerm, exists := existingPermMap[requestedSlug] + if exists { + requestedPermissions = append(requestedPermissions, existingPerm) + continue + } + + newPermID := uid.New(uid.PermissionPrefix) + permissionsToCreate = append(permissionsToCreate, db.InsertPermissionParams{ + PermissionID: newPermID, + WorkspaceID: auth.AuthorizedWorkspaceID, + Name: requestedSlug, + Slug: requestedSlug, + Description: dbtype.NullString{String: "", Valid: false}, + CreatedAtM: now, + }) - for _, requestedName := range *req.Roles { - existingRole, exists := existingRoleMap[requestedName] - if exists { - requestedRoles = append(requestedRoles, existingRole) - continue + requestedPermissions = append(requestedPermissions, db.Permission{ + ID: newPermID, + Name: requestedSlug, + Slug: requestedSlug, + CreatedAtM: now, + WorkspaceID: auth.AuthorizedWorkspaceID, + Description: dbtype.NullString{String: "", Valid: false}, + UpdatedAtM: sql.NullInt64{Int64: 0, Valid: false}, + }) } - return fault.New("role not found", - fault.Code(codes.Data.Role.NotFound.URN()), - fault.Internal("role not found"), fault.Public(fmt.Sprintf("Role '%s' was not found.", requestedName)), - ) - } + if len(permissionsToCreate) > 0 { + err = db.BulkQuery.InsertPermissions(ctx, tx, permissionsToCreate) + if err != nil { + return fault.Wrap(err, + fault.Code(codes.App.Internal.ServiceUnavailable.URN()), + fault.Internal("database error"), + fault.Public("Failed to create permissions."), + ) + } + } - // Insert all requested roles - rolesToInsert := []db.InsertKeyRoleParams{} - for _, reqRole := range requestedRoles { - rolesToInsert = append(rolesToInsert, db.InsertKeyRoleParams{ - KeyID: keyID, - RoleID: reqRole.ID, - WorkspaceID: auth.AuthorizedWorkspaceID, - CreatedAtM: now, - }) + permissionsToInsert := []db.InsertKeyPermissionParams{} + for _, reqPerm := range requestedPermissions { + permissionsToInsert = append(permissionsToInsert, db.InsertKeyPermissionParams{ + KeyID: keyID, + PermissionID: reqPerm.ID, + WorkspaceID: auth.AuthorizedWorkspaceID, + CreatedAt: now, + UpdatedAt: sql.NullInt64{Valid: false, Int64: 0}, + }) - auditLogs = append(auditLogs, auditlog.AuditLog{ - WorkspaceID: auth.AuthorizedWorkspaceID, - Event: auditlog.AuthConnectRoleKeyEvent, - ActorType: auditlog.RootKeyActor, - ActorID: auth.Key.ID, - ActorName: "root key", - ActorMeta: map[string]any{}, - Display: fmt.Sprintf("Connected role %s to key %s", reqRole.Name, keyID), - RemoteIP: s.Location(), - UserAgent: s.UserAgent(), - Resources: []auditlog.AuditLogResource{ - { - Type: auditlog.KeyResourceType, - ID: keyID, - DisplayName: insertKeyParams.Name.String, - Name: insertKeyParams.Name.String, - Meta: map[string]any{}, - }, - { - Type: auditlog.RoleResourceType, - ID: reqRole.ID, - DisplayName: reqRole.Name, - Name: reqRole.Name, - Meta: map[string]any{}, + auditLogs = append(auditLogs, auditlog.AuditLog{ + WorkspaceID: auth.AuthorizedWorkspaceID, + Event: auditlog.AuthConnectPermissionKeyEvent, + ActorType: auditlog.RootKeyActor, + ActorID: auth.Key.ID, + ActorName: "root key", + ActorMeta: map[string]any{}, + Display: fmt.Sprintf("Added permission %s to key %s", reqPerm.Slug, keyID), + RemoteIP: s.Location(), + UserAgent: s.UserAgent(), + Resources: []auditlog.AuditLogResource{ + { + Type: auditlog.KeyResourceType, + ID: keyID, + Name: insertKeyParams.Name.String, + DisplayName: insertKeyParams.Name.String, + Meta: map[string]any{}, + }, + { + Type: auditlog.PermissionResourceType, + ID: reqPerm.ID, + Name: reqPerm.Slug, + DisplayName: reqPerm.Slug, + Meta: map[string]any{}, + }, }, - }, - }) + }) + } + + if len(permissionsToInsert) > 0 { + err = db.BulkQuery.InsertKeyPermissions(ctx, tx, permissionsToInsert) + if err != nil { + return fault.Wrap(err, + fault.Code(codes.App.Internal.ServiceUnavailable.URN()), + fault.Internal("database error"), + fault.Public("Failed to assign permissions."), + ) + } + } } - if len(rolesToInsert) > 0 { - err = db.BulkQuery.InsertKeyRoles(ctx, tx, rolesToInsert) + // 12. Handle roles if provided - with auto-creation + if req.Roles != nil { + existingRoles, err := db.Query.FindRolesByNames(ctx, tx, db.FindRolesByNamesParams{ + WorkspaceID: auth.AuthorizedWorkspaceID, + Names: *req.Roles, + }) if err != nil { return fault.Wrap(err, fault.Code(codes.App.Internal.ServiceUnavailable.URN()), fault.Internal("database error"), - fault.Public("Failed to assign roles."), + fault.Public("Failed to retrieve roles."), + ) + } + + // Find which roles need to be created + existingRoleMap := make(map[string]db.FindRolesByNamesRow) + for _, r := range existingRoles { + existingRoleMap[r.Name] = r + } + + // Create missing roles in bulk and build final list + requestedRoles := []db.FindRolesByNamesRow{} + + for _, requestedName := range *req.Roles { + existingRole, exists := existingRoleMap[requestedName] + if exists { + requestedRoles = append(requestedRoles, existingRole) + continue + } + + return fault.New("role not found", + fault.Code(codes.Data.Role.NotFound.URN()), + fault.Internal("role not found"), fault.Public(fmt.Sprintf("Role '%s' was not found.", requestedName)), ) } + + // Insert all requested roles + rolesToInsert := []db.InsertKeyRoleParams{} + for _, reqRole := range requestedRoles { + rolesToInsert = append(rolesToInsert, db.InsertKeyRoleParams{ + KeyID: keyID, + RoleID: reqRole.ID, + WorkspaceID: auth.AuthorizedWorkspaceID, + CreatedAtM: now, + }) + + auditLogs = append(auditLogs, auditlog.AuditLog{ + WorkspaceID: auth.AuthorizedWorkspaceID, + Event: auditlog.AuthConnectRoleKeyEvent, + ActorType: auditlog.RootKeyActor, + ActorID: auth.Key.ID, + ActorName: "root key", + ActorMeta: map[string]any{}, + Display: fmt.Sprintf("Connected role %s to key %s", reqRole.Name, keyID), + RemoteIP: s.Location(), + UserAgent: s.UserAgent(), + Resources: []auditlog.AuditLogResource{ + { + Type: auditlog.KeyResourceType, + ID: keyID, + DisplayName: insertKeyParams.Name.String, + Name: insertKeyParams.Name.String, + Meta: map[string]any{}, + }, + { + Type: auditlog.RoleResourceType, + ID: reqRole.ID, + DisplayName: reqRole.Name, + Name: reqRole.Name, + Meta: map[string]any{}, + }, + }, + }) + } + + if len(rolesToInsert) > 0 { + err = db.BulkQuery.InsertKeyRoles(ctx, tx, rolesToInsert) + if err != nil { + return fault.Wrap(err, + fault.Code(codes.App.Internal.ServiceUnavailable.URN()), + fault.Internal("database error"), + fault.Public("Failed to assign roles."), + ) + } + } } - } - // 13. Create main audit log for key creation - auditLogs = append(auditLogs, auditlog.AuditLog{ - WorkspaceID: auth.AuthorizedWorkspaceID, - Event: auditlog.KeyCreateEvent, - ActorType: auditlog.RootKeyActor, - ActorID: auth.Key.ID, - ActorName: "root key", - ActorMeta: map[string]any{}, - Display: fmt.Sprintf("Created key %s", keyID), - RemoteIP: s.Location(), - UserAgent: s.UserAgent(), - Resources: []auditlog.AuditLogResource{ - { - Type: auditlog.KeyResourceType, - ID: keyID, - DisplayName: keyID, - Name: keyID, - Meta: map[string]any{}, - }, - { - Type: auditlog.APIResourceType, - ID: req.ApiId, - DisplayName: api.Name, - Name: api.Name, - Meta: map[string]any{}, + // 13. Create main audit log for key creation + auditLogs = append(auditLogs, auditlog.AuditLog{ + WorkspaceID: auth.AuthorizedWorkspaceID, + Event: auditlog.KeyCreateEvent, + ActorType: auditlog.RootKeyActor, + ActorID: auth.Key.ID, + ActorName: "root key", + ActorMeta: map[string]any{}, + Display: fmt.Sprintf("Created key %s", keyID), + RemoteIP: s.Location(), + UserAgent: s.UserAgent(), + Resources: []auditlog.AuditLogResource{ + { + Type: auditlog.KeyResourceType, + ID: keyID, + DisplayName: keyID, + Name: keyID, + Meta: map[string]any{}, + }, + { + Type: auditlog.APIResourceType, + ID: req.ApiId, + DisplayName: api.Name, + Name: api.Name, + Meta: map[string]any{}, + }, }, - }, + }) + + // 14. Insert audit logs + err = h.Auditlogs.Insert(ctx, tx, auditLogs) + if err != nil { + return err + } + + return nil }) - // 14. Insert audit logs - err = h.Auditlogs.Insert(ctx, tx, auditLogs) - if err != nil { - return err + // Break if successful + if txErr == nil { + break } - return nil - }) - if err != nil { - return err + // Check if error is retryable (deadlock or identity race condition) + isRetryable := db.IsDeadlockError(txErr) || (db.IsDuplicateKeyError(txErr) && attempt < 2) + + if !isRetryable { + break + } + } + + if txErr != nil { + // Wrap retryable errors with appropriate message after exhausting retries + if db.IsDuplicateKeyError(txErr) || db.IsDeadlockError(txErr) { + return fault.Wrap(txErr, + fault.Code(codes.App.Internal.ServiceUnavailable.URN()), + fault.Internal("failed to create identity after retries"), + fault.Public("Failed to create identity."), + ) + } + return txErr } // 16. Return success response diff --git a/go/apps/api/routes/v2_keys_update_key/handler.go b/go/apps/api/routes/v2_keys_update_key/handler.go index cd6ba70c75..9ee3a76c2f 100644 --- a/go/apps/api/routes/v2_keys_update_key/handler.go +++ b/go/apps/api/routes/v2_keys_update_key/handler.go @@ -107,511 +107,523 @@ func (h *Handler) Handle(ctx context.Context, s *zen.Session) error { return err } - err = db.Tx(ctx, h.DB.RW(), func(ctx context.Context, tx db.DBTX) error { - auditLogs := []auditlog.AuditLog{} - - update := db.UpdateKeyParams{ - ID: key.ID, - Now: sql.NullInt64{Valid: true, Int64: time.Now().UnixMilli()}, - NameSpecified: 0, - Name: sql.NullString{Valid: false, String: ""}, - IdentityIDSpecified: 0, - IdentityID: sql.NullString{Valid: false, String: ""}, - EnabledSpecified: 0, - Enabled: sql.NullBool{Valid: false, Bool: false}, - MetaSpecified: 0, - Meta: sql.NullString{Valid: false, String: ""}, - ExpiresSpecified: 0, - Expires: sql.NullTime{Valid: false, Time: time.Time{}}, - RemainingRequestsSpecified: 0, - RemainingRequests: sql.NullInt32{Valid: false, Int32: 0}, - RefillAmountSpecified: 0, - RefillAmount: sql.NullInt32{Valid: false, Int32: 0}, - RefillDaySpecified: 0, - RefillDay: sql.NullInt16{Valid: false, Int16: 0}, - } - - if req.Name.IsSpecified() { - update.NameSpecified = 1 - if req.Name.IsNull() { - update.Name = sql.NullString{Valid: false} - } else { - update.Name = sql.NullString{Valid: true, String: req.Name.MustGet()} + // Retry transaction up to 3 times on deadlock or identity creation race + var txErr error + for attempt := 0; attempt < 3; attempt++ { + txErr = db.Tx(ctx, h.DB.RW(), func(ctx context.Context, tx db.DBTX) error { + auditLogs := []auditlog.AuditLog{} + + update := db.UpdateKeyParams{ + ID: key.ID, + Now: sql.NullInt64{Valid: true, Int64: time.Now().UnixMilli()}, + NameSpecified: 0, + Name: sql.NullString{Valid: false, String: ""}, + IdentityIDSpecified: 0, + IdentityID: sql.NullString{Valid: false, String: ""}, + EnabledSpecified: 0, + Enabled: sql.NullBool{Valid: false, Bool: false}, + MetaSpecified: 0, + Meta: sql.NullString{Valid: false, String: ""}, + ExpiresSpecified: 0, + Expires: sql.NullTime{Valid: false, Time: time.Time{}}, + RemainingRequestsSpecified: 0, + RemainingRequests: sql.NullInt32{Valid: false, Int32: 0}, + RefillAmountSpecified: 0, + RefillAmount: sql.NullInt32{Valid: false, Int32: 0}, + RefillDaySpecified: 0, + RefillDay: sql.NullInt16{Valid: false, Int16: 0}, } - } - - if req.ExternalId.IsSpecified() { - update.IdentityIDSpecified = 1 - if req.ExternalId.IsNull() { - update.IdentityID = sql.NullString{Valid: false} - } else { - externalID := req.ExternalId.MustGet() - // Try to find existing identity - identity, err := db.Query.FindIdentity(ctx, tx, db.FindIdentityParams{ - WorkspaceID: auth.AuthorizedWorkspaceID, - Identity: externalID, - Deleted: false, - }) + if req.Name.IsSpecified() { + update.NameSpecified = 1 + if req.Name.IsNull() { + update.Name = sql.NullString{Valid: false} + } else { + update.Name = sql.NullString{Valid: true, String: req.Name.MustGet()} + } + } - if err != nil { - if !db.IsNotFound(err) { - return fault.Wrap(err, - fault.Code(codes.App.Internal.ServiceUnavailable.URN()), - fault.Internal("failed to find identity"), - fault.Public("Failed to find identity."), - ) - } + if req.ExternalId.IsSpecified() { + update.IdentityIDSpecified = 1 + if req.ExternalId.IsNull() { + update.IdentityID = sql.NullString{Valid: false} + } else { + externalID := req.ExternalId.MustGet() - // Create new identity - identityID := uid.New(uid.IdentityPrefix) - err = db.Query.InsertIdentity(ctx, tx, db.InsertIdentityParams{ - ID: identityID, - ExternalID: externalID, + // Try to find existing identity + identity, err := db.Query.FindIdentity(ctx, tx, db.FindIdentityParams{ WorkspaceID: auth.AuthorizedWorkspaceID, - Environment: "default", - CreatedAt: time.Now().UnixMilli(), - Meta: []byte("{}"), + Identity: externalID, + Deleted: false, }) + if err != nil { - // Incase of duplicate key error just find existing identity - if !db.IsDuplicateKeyError(err) { + if !db.IsNotFound(err) { return fault.Wrap(err, fault.Code(codes.App.Internal.ServiceUnavailable.URN()), - fault.Internal("failed to create identity"), - fault.Public("Failed to create identity."), + fault.Internal("failed to find identity"), + fault.Public("Failed to find identity."), ) } - identity, err = db.Query.FindIdentity(ctx, h.DB.RO(), db.FindIdentityParams{ + // Create new identity + identityID := uid.New(uid.IdentityPrefix) + err = db.Query.InsertIdentity(ctx, tx, db.InsertIdentityParams{ + ID: identityID, + ExternalID: externalID, WorkspaceID: auth.AuthorizedWorkspaceID, - Identity: externalID, - Deleted: false, + Environment: "default", + CreatedAt: time.Now().UnixMilli(), + Meta: []byte("{}"), }) if err != nil { + // Don't wrap duplicate key or deadlock errors - let retry loop handle them + if db.IsDuplicateKeyError(err) || db.IsDeadlockError(err) { + return err + } + return fault.Wrap(err, fault.Code(codes.App.Internal.ServiceUnavailable.URN()), - fault.Internal("failed to find identity"), - fault.Public("Failed to find identity."), + fault.Internal("failed to create identity"), + fault.Public("Failed to create identity."), ) } - identityID = identity.ID + update.IdentityID = sql.NullString{Valid: true, String: identityID} + } else { + // Use existing identity + update.IdentityID = sql.NullString{Valid: true, String: identity.ID} } - - update.IdentityID = sql.NullString{Valid: true, String: identityID} - } else { - // Use existing identity - update.IdentityID = sql.NullString{Valid: true, String: identity.ID} } } - } - if req.Enabled != nil { - update.EnabledSpecified = 1 - update.Enabled = sql.NullBool{Valid: true, Bool: *req.Enabled} - } + if req.Enabled != nil { + update.EnabledSpecified = 1 + update.Enabled = sql.NullBool{Valid: true, Bool: *req.Enabled} + } - if req.Meta.IsSpecified() { - update.MetaSpecified = 1 - if req.Meta.IsNull() { - update.Meta = sql.NullString{Valid: false} - } else { - metaBytes, marshalErr := json.Marshal(req.Meta.MustGet()) - if marshalErr != nil { - return fault.Wrap(marshalErr, - fault.Code(codes.App.Validation.InvalidInput.URN()), - fault.Internal("failed to marshal meta"), - fault.Public("Invalid metadata format."), - ) + if req.Meta.IsSpecified() { + update.MetaSpecified = 1 + if req.Meta.IsNull() { + update.Meta = sql.NullString{Valid: false} + } else { + metaBytes, marshalErr := json.Marshal(req.Meta.MustGet()) + if marshalErr != nil { + return fault.Wrap(marshalErr, + fault.Code(codes.App.Validation.InvalidInput.URN()), + fault.Internal("failed to marshal meta"), + fault.Public("Invalid metadata format."), + ) + } + update.Meta = sql.NullString{Valid: true, String: string(metaBytes)} } - update.Meta = sql.NullString{Valid: true, String: string(metaBytes)} } - } - if req.Expires.IsSpecified() { - update.ExpiresSpecified = 1 - if req.Expires.IsNull() { - update.Expires = sql.NullTime{Valid: false} - } else { - update.Expires = sql.NullTime{Valid: true, Time: time.UnixMilli(req.Expires.MustGet())} + if req.Expires.IsSpecified() { + update.ExpiresSpecified = 1 + if req.Expires.IsNull() { + update.Expires = sql.NullTime{Valid: false} + } else { + update.Expires = sql.NullTime{Valid: true, Time: time.UnixMilli(req.Expires.MustGet())} + } } - } - if req.Credits.IsSpecified() { - if req.Credits.IsNull() { - update.RemainingRequestsSpecified = 1 - update.RefillAmountSpecified = 1 - update.RefillDaySpecified = 1 - update.RefillAmount = sql.NullInt32{Valid: false, Int32: 0} - update.RefillDay = sql.NullInt16{Valid: false, Int16: 0} - update.RemainingRequests = sql.NullInt32{Valid: false} - } else { - credits := req.Credits.MustGet() - if credits.Remaining.IsSpecified() { + if req.Credits.IsSpecified() { + if req.Credits.IsNull() { update.RemainingRequestsSpecified = 1 - if credits.Remaining.IsNull() { - // This also clears refilling - update.RefillAmountSpecified = 1 - update.RefillDaySpecified = 1 - update.RemainingRequests = sql.NullInt32{Valid: false} - update.RefillAmount = sql.NullInt32{Valid: false, Int32: 0} - update.RefillDay = sql.NullInt16{Valid: false, Int16: 0} - } else { - update.RemainingRequests = sql.NullInt32{ - Valid: true, - Int32: int32(credits.Remaining.MustGet()), // nolint:gosec + update.RefillAmountSpecified = 1 + update.RefillDaySpecified = 1 + update.RefillAmount = sql.NullInt32{Valid: false, Int32: 0} + update.RefillDay = sql.NullInt16{Valid: false, Int16: 0} + update.RemainingRequests = sql.NullInt32{Valid: false} + } else { + credits := req.Credits.MustGet() + if credits.Remaining.IsSpecified() { + update.RemainingRequestsSpecified = 1 + if credits.Remaining.IsNull() { + // This also clears refilling + update.RefillAmountSpecified = 1 + update.RefillDaySpecified = 1 + update.RemainingRequests = sql.NullInt32{Valid: false} + update.RefillAmount = sql.NullInt32{Valid: false, Int32: 0} + update.RefillDay = sql.NullInt16{Valid: false, Int16: 0} + } else { + update.RemainingRequests = sql.NullInt32{ + Valid: true, + Int32: int32(credits.Remaining.MustGet()), // nolint:gosec + } } } - } - if credits.Refill.IsSpecified() { - if credits.Refill.IsNull() { - update.RefillAmountSpecified = 1 - update.RefillDaySpecified = 1 - update.RefillAmount = sql.NullInt32{Valid: false, Int32: 0} - update.RefillDay = sql.NullInt16{Valid: false, Int16: 0} - } else { - refill := credits.Refill.MustGet() - update.RefillAmountSpecified = 1 - update.RefillAmount = sql.NullInt32{ - Valid: true, - Int32: int32(refill.Amount), // nolint:gosec - } - - update.RefillDaySpecified = 1 - switch refill.Interval { - case openapi.UpdateKeyCreditsRefillIntervalMonthly: - if refill.RefillDay == nil { - return fault.New("missing refillDay", - fault.Code(codes.App.Validation.InvalidInput.URN()), - fault.Internal("refillDay required for monthly interval"), - fault.Public("`refillDay` must be provided when the refill interval is `monthly`."), - ) - } - - update.RefillDay = sql.NullInt16{ + if credits.Refill.IsSpecified() { + if credits.Refill.IsNull() { + update.RefillAmountSpecified = 1 + update.RefillDaySpecified = 1 + update.RefillAmount = sql.NullInt32{Valid: false, Int32: 0} + update.RefillDay = sql.NullInt16{Valid: false, Int16: 0} + } else { + refill := credits.Refill.MustGet() + update.RefillAmountSpecified = 1 + update.RefillAmount = sql.NullInt32{ Valid: true, - Int16: int16(*refill.RefillDay), // nolint:gosec - } - case openapi.UpdateKeyCreditsRefillIntervalDaily: - if refill.RefillDay != nil { - return fault.New("invalid refillDay", - fault.Code(codes.App.Validation.InvalidInput.URN()), - fault.Internal("refillDay cannot be set for daily interval"), - fault.Public("`refillDay` must not be provided when the refill interval is `daily`."), - ) + Int32: int32(refill.Amount), // nolint:gosec } - // For daily, refill_day should remain NULL - update.RefillDay = sql.NullInt16{Valid: false} + update.RefillDaySpecified = 1 + switch refill.Interval { + case openapi.UpdateKeyCreditsRefillIntervalMonthly: + if refill.RefillDay == nil { + return fault.New("missing refillDay", + fault.Code(codes.App.Validation.InvalidInput.URN()), + fault.Internal("refillDay required for monthly interval"), + fault.Public("`refillDay` must be provided when the refill interval is `monthly`."), + ) + } + + update.RefillDay = sql.NullInt16{ + Valid: true, + Int16: int16(*refill.RefillDay), // nolint:gosec + } + case openapi.UpdateKeyCreditsRefillIntervalDaily: + if refill.RefillDay != nil { + return fault.New("invalid refillDay", + fault.Code(codes.App.Validation.InvalidInput.URN()), + fault.Internal("refillDay cannot be set for daily interval"), + fault.Public("`refillDay` must not be provided when the refill interval is `daily`."), + ) + } + + // For daily, refill_day should remain NULL + update.RefillDay = sql.NullInt16{Valid: false} + } } } } } - } - err = db.Query.UpdateKey(ctx, tx, update) - if err != nil { - return fault.Wrap(err, - fault.Code(codes.App.Internal.ServiceUnavailable.URN()), - fault.Internal("database error"), - fault.Public("Failed to update key."), - ) - } - - if req.Ratelimits != nil { - existingRatelimits, err := db.Query.ListRatelimitsByKeyID(ctx, tx, sql.NullString{String: key.ID, Valid: true}) - if err != nil && !db.IsNotFound(err) { + err = db.Query.UpdateKey(ctx, tx, update) + if err != nil { return fault.Wrap(err, - fault.Internal("unable to fetch ratelimits"), - fault.Public("Failed to retrieve key ratelimits."), + fault.Code(codes.App.Internal.ServiceUnavailable.URN()), + fault.Internal("database error"), + fault.Public("Failed to update key."), ) } - // Create map of existing ratelimits by name - existingRatelimitMap := make(map[string]db.ListRatelimitsByKeyIDRow) - for _, rl := range existingRatelimits { - existingRatelimitMap[rl.Name] = rl - } + if req.Ratelimits != nil { + existingRatelimits, err := db.Query.ListRatelimitsByKeyID(ctx, tx, sql.NullString{String: key.ID, Valid: true}) + if err != nil && !db.IsNotFound(err) { + return fault.Wrap(err, + fault.Internal("unable to fetch ratelimits"), + fault.Public("Failed to retrieve key ratelimits."), + ) + } - // Create map of new ratelimits - newRatelimitMap := make(map[string]openapi.RatelimitRequest) - for _, rl := range *req.Ratelimits { - newRatelimitMap[rl.Name] = rl - } + // Create map of existing ratelimits by name + existingRatelimitMap := make(map[string]db.ListRatelimitsByKeyIDRow) + for _, rl := range existingRatelimits { + existingRatelimitMap[rl.Name] = rl + } - // Delete ratelimits that are not in the new list - rateLimitsToDelete := []string{} - for _, existingRL := range existingRatelimits { - if _, exists := newRatelimitMap[existingRL.Name]; !exists { - rateLimitsToDelete = append(rateLimitsToDelete, existingRL.ID) + // Create map of new ratelimits + newRatelimitMap := make(map[string]openapi.RatelimitRequest) + for _, rl := range *req.Ratelimits { + newRatelimitMap[rl.Name] = rl } - } - if len(rateLimitsToDelete) > 0 { - err = db.Query.DeleteManyRatelimitsByIDs(ctx, tx, rateLimitsToDelete) - if err != nil { - return fault.Wrap(err, - fault.Internal("unable to delete ratelimits"), - fault.Public("Failed to delete ratelimits."), - ) + // Delete ratelimits that are not in the new list + rateLimitsToDelete := []string{} + for _, existingRL := range existingRatelimits { + if _, exists := newRatelimitMap[existingRL.Name]; !exists { + rateLimitsToDelete = append(rateLimitsToDelete, existingRL.ID) + } + } + + if len(rateLimitsToDelete) > 0 { + err = db.Query.DeleteManyRatelimitsByIDs(ctx, tx, rateLimitsToDelete) + if err != nil { + return fault.Wrap(err, + fault.Internal("unable to delete ratelimits"), + fault.Public("Failed to delete ratelimits."), + ) + } } - } - // Insert or update ratelimits - ratelimitsToInsert := []db.InsertKeyRatelimitParams{} - now := time.Now().UnixMilli() - for name, newRL := range newRatelimitMap { - _, exists := existingRatelimitMap[name] + // Insert or update ratelimits + ratelimitsToInsert := []db.InsertKeyRatelimitParams{} + now := time.Now().UnixMilli() + for name, newRL := range newRatelimitMap { + _, exists := existingRatelimitMap[name] - var rlID string - if exists { - rlID = existingRatelimitMap[name].ID - } else { - rlID = uid.New(uid.RatelimitPrefix) + var rlID string + if exists { + rlID = existingRatelimitMap[name].ID + } else { + rlID = uid.New(uid.RatelimitPrefix) + } + + ratelimitsToInsert = append(ratelimitsToInsert, db.InsertKeyRatelimitParams{ + ID: rlID, + WorkspaceID: auth.AuthorizedWorkspaceID, + KeyID: sql.NullString{String: key.ID, Valid: true}, + Name: newRL.Name, + Limit: int32(newRL.Limit), // nolint:gosec + Duration: newRL.Duration, + CreatedAt: now, + UpdatedAt: sql.NullInt64{Int64: now, Valid: true}, + AutoApply: newRL.AutoApply, + }) } - ratelimitsToInsert = append(ratelimitsToInsert, db.InsertKeyRatelimitParams{ - ID: rlID, - WorkspaceID: auth.AuthorizedWorkspaceID, - KeyID: sql.NullString{String: key.ID, Valid: true}, - Name: newRL.Name, - Limit: int32(newRL.Limit), // nolint:gosec - Duration: newRL.Duration, - CreatedAt: now, - UpdatedAt: sql.NullInt64{Int64: now, Valid: true}, - AutoApply: newRL.AutoApply, - }) + if len(ratelimitsToInsert) > 0 { + err = db.BulkQuery.InsertKeyRatelimits(ctx, tx, ratelimitsToInsert) + if err != nil { + return fault.Wrap(err, + fault.Code(codes.App.Internal.ServiceUnavailable.URN()), + fault.Internal("database error"), + fault.Public("Failed to update rate limits."), + ) + } + } } - if len(ratelimitsToInsert) > 0 { - err = db.BulkQuery.InsertKeyRatelimits(ctx, tx, ratelimitsToInsert) + if req.Permissions != nil { + existingPermissions, err := db.Query.FindPermissionsBySlugs(ctx, tx, db.FindPermissionsBySlugsParams{ + WorkspaceID: auth.AuthorizedWorkspaceID, + Slugs: *req.Permissions, + }) if err != nil { return fault.Wrap(err, fault.Code(codes.App.Internal.ServiceUnavailable.URN()), fault.Internal("database error"), - fault.Public("Failed to update rate limits."), + fault.Public("Failed to retrieve permissions."), ) } - } - } - if req.Permissions != nil { - existingPermissions, err := db.Query.FindPermissionsBySlugs(ctx, tx, db.FindPermissionsBySlugsParams{ - WorkspaceID: auth.AuthorizedWorkspaceID, - Slugs: *req.Permissions, - }) - if err != nil { - return fault.Wrap(err, - fault.Code(codes.App.Internal.ServiceUnavailable.URN()), - fault.Internal("database error"), - fault.Public("Failed to retrieve permissions."), - ) - } - - existingPermMap := make(map[string]db.Permission) - for _, p := range existingPermissions { - existingPermMap[p.Slug] = p - } + existingPermMap := make(map[string]db.Permission) + for _, p := range existingPermissions { + existingPermMap[p.Slug] = p + } - permissionsToCreate := []db.InsertPermissionParams{} - requestedPermissions := []db.Permission{} + permissionsToCreate := []db.InsertPermissionParams{} + requestedPermissions := []db.Permission{} - for _, requestedSlug := range *req.Permissions { - existingPerm, exists := existingPermMap[requestedSlug] - if exists { - requestedPermissions = append(requestedPermissions, existingPerm) - continue - } + for _, requestedSlug := range *req.Permissions { + existingPerm, exists := existingPermMap[requestedSlug] + if exists { + requestedPermissions = append(requestedPermissions, existingPerm) + continue + } - newPermID := uid.New(uid.PermissionPrefix) - permissionsToCreate = append(permissionsToCreate, db.InsertPermissionParams{ - PermissionID: newPermID, - WorkspaceID: auth.AuthorizedWorkspaceID, - Name: requestedSlug, - Slug: requestedSlug, - Description: dbtype.NullString{String: fmt.Sprintf("Auto-created permission: %s", requestedSlug), Valid: true}, - CreatedAtM: time.Now().UnixMilli(), - }) + newPermID := uid.New(uid.PermissionPrefix) + permissionsToCreate = append(permissionsToCreate, db.InsertPermissionParams{ + PermissionID: newPermID, + WorkspaceID: auth.AuthorizedWorkspaceID, + Name: requestedSlug, + Slug: requestedSlug, + Description: dbtype.NullString{String: fmt.Sprintf("Auto-created permission: %s", requestedSlug), Valid: true}, + CreatedAtM: time.Now().UnixMilli(), + }) - requestedPermissions = append(requestedPermissions, db.Permission{ - ID: newPermID, - Slug: requestedSlug, - }) - } + requestedPermissions = append(requestedPermissions, db.Permission{ + ID: newPermID, + Slug: requestedSlug, + }) + } - if len(permissionsToCreate) > 0 { - for _, toCreate := range permissionsToCreate { - auditLogs = append(auditLogs, auditlog.AuditLog{ - WorkspaceID: auth.AuthorizedWorkspaceID, - Event: auditlog.PermissionCreateEvent, - ActorType: auditlog.RootKeyActor, - ActorID: auth.Key.ID, - ActorName: "root key", - ActorMeta: map[string]any{}, - Display: fmt.Sprintf("Created %s (%s)", toCreate.Slug, toCreate.PermissionID), - RemoteIP: s.Location(), - UserAgent: s.UserAgent(), - Resources: []auditlog.AuditLogResource{ - { - Type: auditlog.PermissionResourceType, - ID: toCreate.PermissionID, - Name: toCreate.Slug, - DisplayName: toCreate.Name, - Meta: map[string]interface{}{ - "name": toCreate.Name, - "slug": toCreate.Slug, + if len(permissionsToCreate) > 0 { + for _, toCreate := range permissionsToCreate { + auditLogs = append(auditLogs, auditlog.AuditLog{ + WorkspaceID: auth.AuthorizedWorkspaceID, + Event: auditlog.PermissionCreateEvent, + ActorType: auditlog.RootKeyActor, + ActorID: auth.Key.ID, + ActorName: "root key", + ActorMeta: map[string]any{}, + Display: fmt.Sprintf("Created %s (%s)", toCreate.Slug, toCreate.PermissionID), + RemoteIP: s.Location(), + UserAgent: s.UserAgent(), + Resources: []auditlog.AuditLogResource{ + { + Type: auditlog.PermissionResourceType, + ID: toCreate.PermissionID, + Name: toCreate.Slug, + DisplayName: toCreate.Name, + Meta: map[string]interface{}{ + "name": toCreate.Name, + "slug": toCreate.Slug, + }, }, }, - }, - }) + }) + } + + err = db.BulkQuery.InsertPermissions(ctx, tx, permissionsToCreate) + if err != nil { + return fault.Wrap(err, + fault.Code(codes.App.Internal.ServiceUnavailable.URN()), + fault.Internal("database error"), + fault.Public("Failed to create permissions."), + ) + } } - err = db.BulkQuery.InsertPermissions(ctx, tx, permissionsToCreate) + err = db.Query.DeleteAllKeyPermissionsByKeyID(ctx, tx, key.ID) if err != nil { return fault.Wrap(err, - fault.Code(codes.App.Internal.ServiceUnavailable.URN()), - fault.Internal("database error"), - fault.Public("Failed to create permissions."), + fault.Internal("unable to clear permissions"), + fault.Public("Failed to clear key permissions."), ) } - } - err = db.Query.DeleteAllKeyPermissionsByKeyID(ctx, tx, key.ID) - if err != nil { - return fault.Wrap(err, - fault.Internal("unable to clear permissions"), - fault.Public("Failed to clear key permissions."), - ) - } + permissionsToInsert := []db.InsertKeyPermissionParams{} + now := time.Now().UnixMilli() + for _, reqPerm := range requestedPermissions { + permissionsToInsert = append(permissionsToInsert, db.InsertKeyPermissionParams{ + KeyID: key.ID, + PermissionID: reqPerm.ID, + WorkspaceID: auth.AuthorizedWorkspaceID, + CreatedAt: now, + UpdatedAt: sql.NullInt64{Int64: now, Valid: true}, + }) + } - permissionsToInsert := []db.InsertKeyPermissionParams{} - now := time.Now().UnixMilli() - for _, reqPerm := range requestedPermissions { - permissionsToInsert = append(permissionsToInsert, db.InsertKeyPermissionParams{ - KeyID: key.ID, - PermissionID: reqPerm.ID, - WorkspaceID: auth.AuthorizedWorkspaceID, - CreatedAt: now, - UpdatedAt: sql.NullInt64{Int64: now, Valid: true}, - }) + if len(permissionsToInsert) > 0 { + err = db.BulkQuery.InsertKeyPermissions(ctx, tx, permissionsToInsert) + if err != nil { + return fault.Wrap(err, + fault.Code(codes.App.Internal.ServiceUnavailable.URN()), + fault.Internal("database error"), + fault.Public("Failed to assign permissions."), + ) + } + } } - if len(permissionsToInsert) > 0 { - err = db.BulkQuery.InsertKeyPermissions(ctx, tx, permissionsToInsert) + if req.Roles != nil { + existingRoles, err := db.Query.FindRolesByNames(ctx, tx, db.FindRolesByNamesParams{ + WorkspaceID: auth.AuthorizedWorkspaceID, + Names: *req.Roles, + }) if err != nil { return fault.Wrap(err, fault.Code(codes.App.Internal.ServiceUnavailable.URN()), fault.Internal("database error"), - fault.Public("Failed to assign permissions."), + fault.Public("Failed to retrieve roles."), ) } - } - } - - if req.Roles != nil { - existingRoles, err := db.Query.FindRolesByNames(ctx, tx, db.FindRolesByNamesParams{ - WorkspaceID: auth.AuthorizedWorkspaceID, - Names: *req.Roles, - }) - if err != nil { - return fault.Wrap(err, - fault.Code(codes.App.Internal.ServiceUnavailable.URN()), - fault.Internal("database error"), - fault.Public("Failed to retrieve roles."), - ) - } - // Find which roles need to be created - existingRoleMap := make(map[string]db.FindRolesByNamesRow) - for _, r := range existingRoles { - existingRoleMap[r.Name] = r - } - - requestedRoles := []db.FindRolesByNamesRow{} - for _, requestedName := range *req.Roles { - existingRole, exists := existingRoleMap[requestedName] - if exists { - requestedRoles = append(requestedRoles, existingRole) - continue + // Find which roles need to be created + existingRoleMap := make(map[string]db.FindRolesByNamesRow) + for _, r := range existingRoles { + existingRoleMap[r.Name] = r } - return fault.New("role not found", - fault.Code(codes.Data.Role.NotFound.URN()), - fault.Internal("role not found"), - fault.Public(fmt.Sprintf("Role '%s' was not found.", requestedName)), - ) - } - - err = db.Query.DeleteAllKeyRolesByKeyID(ctx, tx, key.ID) - if err != nil { - return fault.Wrap(err, - fault.Internal("unable to clear roles"), - fault.Public("Failed to clear key roles."), - ) - } + requestedRoles := []db.FindRolesByNamesRow{} + for _, requestedName := range *req.Roles { + existingRole, exists := existingRoleMap[requestedName] + if exists { + requestedRoles = append(requestedRoles, existingRole) + continue + } - // Insert all requested roles - rolesToInsert := []db.InsertKeyRoleParams{} - for _, reqRole := range requestedRoles { - rolesToInsert = append(rolesToInsert, db.InsertKeyRoleParams{ - KeyID: key.ID, - RoleID: reqRole.ID, - WorkspaceID: auth.AuthorizedWorkspaceID, - CreatedAtM: time.Now().UnixMilli(), - }) - } + return fault.New("role not found", + fault.Code(codes.Data.Role.NotFound.URN()), + fault.Internal("role not found"), + fault.Public(fmt.Sprintf("Role '%s' was not found.", requestedName)), + ) + } - if len(rolesToInsert) > 0 { - err = db.BulkQuery.InsertKeyRoles(ctx, tx, rolesToInsert) + err = db.Query.DeleteAllKeyRolesByKeyID(ctx, tx, key.ID) if err != nil { return fault.Wrap(err, - fault.Code(codes.App.Internal.ServiceUnavailable.URN()), - fault.Internal("database error"), - fault.Public("Failed to assign roles."), + fault.Internal("unable to clear roles"), + fault.Public("Failed to clear key roles."), ) } + + // Insert all requested roles + rolesToInsert := []db.InsertKeyRoleParams{} + for _, reqRole := range requestedRoles { + rolesToInsert = append(rolesToInsert, db.InsertKeyRoleParams{ + KeyID: key.ID, + RoleID: reqRole.ID, + WorkspaceID: auth.AuthorizedWorkspaceID, + CreatedAtM: time.Now().UnixMilli(), + }) + } + + if len(rolesToInsert) > 0 { + err = db.BulkQuery.InsertKeyRoles(ctx, tx, rolesToInsert) + if err != nil { + return fault.Wrap(err, + fault.Code(codes.App.Internal.ServiceUnavailable.URN()), + fault.Internal("database error"), + fault.Public("Failed to assign roles."), + ) + } + } } - } - auditLogs = append(auditLogs, auditlog.AuditLog{ - WorkspaceID: auth.AuthorizedWorkspaceID, - Event: auditlog.KeyUpdateEvent, - ActorType: auditlog.RootKeyActor, - ActorID: auth.Key.ID, - ActorName: "root key", - ActorMeta: map[string]any{}, - Display: fmt.Sprintf("Updated key %s", key.ID), - RemoteIP: s.Location(), - UserAgent: s.UserAgent(), - Resources: []auditlog.AuditLogResource{ - { - Type: auditlog.KeyResourceType, - ID: key.ID, - DisplayName: key.Name.String, - Name: key.Name.String, - Meta: map[string]any{}, - }, - { - Type: auditlog.APIResourceType, - ID: key.Api.ID, - DisplayName: key.Api.Name, - Name: key.Api.Name, - Meta: map[string]any{}, + auditLogs = append(auditLogs, auditlog.AuditLog{ + WorkspaceID: auth.AuthorizedWorkspaceID, + Event: auditlog.KeyUpdateEvent, + ActorType: auditlog.RootKeyActor, + ActorID: auth.Key.ID, + ActorName: "root key", + ActorMeta: map[string]any{}, + Display: fmt.Sprintf("Updated key %s", key.ID), + RemoteIP: s.Location(), + UserAgent: s.UserAgent(), + Resources: []auditlog.AuditLogResource{ + { + Type: auditlog.KeyResourceType, + ID: key.ID, + DisplayName: key.Name.String, + Name: key.Name.String, + Meta: map[string]any{}, + }, + { + Type: auditlog.APIResourceType, + ID: key.Api.ID, + DisplayName: key.Api.Name, + Name: key.Api.Name, + Meta: map[string]any{}, + }, }, - }, + }) + + err = h.Auditlogs.Insert(ctx, tx, auditLogs) + if err != nil { + return err + } + + return nil }) - err = h.Auditlogs.Insert(ctx, tx, auditLogs) - if err != nil { - return err + // Break if successful + if txErr == nil { + break } - return nil - }) - if err != nil { - return err + // Check if error is retryable (deadlock or identity race condition) + isRetryable := db.IsDeadlockError(txErr) || (db.IsDuplicateKeyError(txErr) && attempt < 2) + + if !isRetryable { + break + } + } + + if txErr != nil { + // Wrap retryable errors with appropriate message after exhausting retries + if db.IsDuplicateKeyError(txErr) || db.IsDeadlockError(txErr) { + return fault.Wrap(txErr, + fault.Code(codes.App.Internal.ServiceUnavailable.URN()), + fault.Internal("failed to create identity after retries"), + fault.Public("Failed to create identity."), + ) + } + return txErr } h.KeyCache.Remove(ctx, key.Hash) diff --git a/go/pkg/db/handle_err_deadlock.go b/go/pkg/db/handle_err_deadlock.go new file mode 100644 index 0000000000..c74d97e226 --- /dev/null +++ b/go/pkg/db/handle_err_deadlock.go @@ -0,0 +1,16 @@ +package db + +import ( + "errors" + + "github.com/go-sql-driver/mysql" +) + +func IsDeadlockError(err error) bool { + var mysqlErr *mysql.MySQLError + if errors.As(err, &mysqlErr) && mysqlErr.Number == 1213 { + return true + } + + return false +} From faffe427c4056cd4dd9b6239cc530c00a6219bff Mon Sep 17 00:00:00 2001 From: Flo Date: Mon, 6 Oct 2025 15:57:00 +0200 Subject: [PATCH 4/7] fix: use range --- go/apps/api/routes/v2_keys_create_key/handler.go | 4 ++-- go/apps/api/routes/v2_keys_update_key/handler.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go/apps/api/routes/v2_keys_create_key/handler.go b/go/apps/api/routes/v2_keys_create_key/handler.go index a8b211b9f1..10475f4063 100644 --- a/go/apps/api/routes/v2_keys_create_key/handler.go +++ b/go/apps/api/routes/v2_keys_create_key/handler.go @@ -178,9 +178,9 @@ func (h *Handler) Handle(ctx context.Context, s *zen.Session) error { now := time.Now().UnixMilli() - // Retry transaction up to 3 times on deadlock or identity creation race + // Retry transaction up to 2 times on deadlock or identity creation race var txErr error - for attempt := 0; attempt < 3; attempt++ { + for attempt := range 2 { txErr = db.Tx(ctx, h.DB.RW(), func(ctx context.Context, tx db.DBTX) error { insertKeyParams := db.InsertKeyParams{ ID: keyID, diff --git a/go/apps/api/routes/v2_keys_update_key/handler.go b/go/apps/api/routes/v2_keys_update_key/handler.go index 9ee3a76c2f..e8efecc9b4 100644 --- a/go/apps/api/routes/v2_keys_update_key/handler.go +++ b/go/apps/api/routes/v2_keys_update_key/handler.go @@ -107,9 +107,9 @@ func (h *Handler) Handle(ctx context.Context, s *zen.Session) error { return err } - // Retry transaction up to 3 times on deadlock or identity creation race + // Retry transaction up to 2 times on deadlock or identity creation race var txErr error - for attempt := 0; attempt < 3; attempt++ { + for attempt := range 2 { txErr = db.Tx(ctx, h.DB.RW(), func(ctx context.Context, tx db.DBTX) error { auditLogs := []auditlog.AuditLog{} From 6032222644c248ddd563d8b7753fdf4805ae7c1c Mon Sep 17 00:00:00 2001 From: Flo Date: Mon, 6 Oct 2025 16:17:06 +0200 Subject: [PATCH 5/7] fix: use range --- go/apps/api/routes/v2_keys_create_key/handler.go | 2 +- go/apps/api/routes/v2_keys_update_key/handler.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/go/apps/api/routes/v2_keys_create_key/handler.go b/go/apps/api/routes/v2_keys_create_key/handler.go index 10475f4063..94841b83ff 100644 --- a/go/apps/api/routes/v2_keys_create_key/handler.go +++ b/go/apps/api/routes/v2_keys_create_key/handler.go @@ -180,7 +180,7 @@ func (h *Handler) Handle(ctx context.Context, s *zen.Session) error { // Retry transaction up to 2 times on deadlock or identity creation race var txErr error - for attempt := range 2 { + for attempt := range 3 { txErr = db.Tx(ctx, h.DB.RW(), func(ctx context.Context, tx db.DBTX) error { insertKeyParams := db.InsertKeyParams{ ID: keyID, diff --git a/go/apps/api/routes/v2_keys_update_key/handler.go b/go/apps/api/routes/v2_keys_update_key/handler.go index e8efecc9b4..582aa966aa 100644 --- a/go/apps/api/routes/v2_keys_update_key/handler.go +++ b/go/apps/api/routes/v2_keys_update_key/handler.go @@ -109,7 +109,7 @@ func (h *Handler) Handle(ctx context.Context, s *zen.Session) error { // Retry transaction up to 2 times on deadlock or identity creation race var txErr error - for attempt := range 2 { + for attempt := range 3 { txErr = db.Tx(ctx, h.DB.RW(), func(ctx context.Context, tx db.DBTX) error { auditLogs := []auditlog.AuditLog{} From e3c6781d88c498939a02c58f6a1c2ae8b61d1f4b Mon Sep 17 00:00:00 2001 From: Flo Date: Tue, 7 Oct 2025 11:45:54 +0200 Subject: [PATCH 6/7] use correct err msg --- go/apps/api/routes/v2_keys_create_key/handler.go | 10 ++-------- go/apps/api/routes/v2_keys_update_key/handler.go | 4 ++-- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/go/apps/api/routes/v2_keys_create_key/handler.go b/go/apps/api/routes/v2_keys_create_key/handler.go index 94841b83ff..58c5a038db 100644 --- a/go/apps/api/routes/v2_keys_create_key/handler.go +++ b/go/apps/api/routes/v2_keys_create_key/handler.go @@ -254,7 +254,6 @@ func (h *Handler) Handle(ctx context.Context, s *zen.Session) error { } } - // Note: owner_id is set to null in the SQL query, so we skip setting it here if req.Meta != nil { metaBytes, marshalErr := json.Marshal(*req.Meta) if marshalErr != nil { @@ -356,7 +355,6 @@ func (h *Handler) Handle(ctx context.Context, s *zen.Session) error { } } - // 11. Handle permissions if provided - with auto-creation var auditLogs []auditlog.AuditLog if req.Permissions != nil { existingPermissions, err := db.Query.FindPermissionsBySlugs(ctx, tx, db.FindPermissionsBySlugsParams{ @@ -469,7 +467,6 @@ func (h *Handler) Handle(ctx context.Context, s *zen.Session) error { } } - // 12. Handle roles if provided - with auto-creation if req.Roles != nil { existingRoles, err := db.Query.FindRolesByNames(ctx, tx, db.FindRolesByNamesParams{ WorkspaceID: auth.AuthorizedWorkspaceID, @@ -556,7 +553,6 @@ func (h *Handler) Handle(ctx context.Context, s *zen.Session) error { } } - // 13. Create main audit log for key creation auditLogs = append(auditLogs, auditlog.AuditLog{ WorkspaceID: auth.AuthorizedWorkspaceID, Event: auditlog.KeyCreateEvent, @@ -585,7 +581,6 @@ func (h *Handler) Handle(ctx context.Context, s *zen.Session) error { }, }) - // 14. Insert audit logs err = h.Auditlogs.Insert(ctx, tx, auditLogs) if err != nil { return err @@ -612,14 +607,13 @@ func (h *Handler) Handle(ctx context.Context, s *zen.Session) error { if db.IsDuplicateKeyError(txErr) || db.IsDeadlockError(txErr) { return fault.Wrap(txErr, fault.Code(codes.App.Internal.ServiceUnavailable.URN()), - fault.Internal("failed to create identity after retries"), - fault.Public("Failed to create identity."), + fault.Internal("failed to create key after retries"), + fault.Public("Failed to create key."), ) } return txErr } - // 16. Return success response return s.JSON(http.StatusOK, Response{ Meta: openapi.Meta{ RequestId: s.RequestID(), diff --git a/go/apps/api/routes/v2_keys_update_key/handler.go b/go/apps/api/routes/v2_keys_update_key/handler.go index 582aa966aa..430d8f4415 100644 --- a/go/apps/api/routes/v2_keys_update_key/handler.go +++ b/go/apps/api/routes/v2_keys_update_key/handler.go @@ -619,8 +619,8 @@ func (h *Handler) Handle(ctx context.Context, s *zen.Session) error { if db.IsDuplicateKeyError(txErr) || db.IsDeadlockError(txErr) { return fault.Wrap(txErr, fault.Code(codes.App.Internal.ServiceUnavailable.URN()), - fault.Internal("failed to create identity after retries"), - fault.Public("Failed to create identity."), + fault.Internal("failed to create key after retries"), + fault.Public("Failed to create key."), ) } return txErr From 90d96faf84215241a0616368dc0ae5c008e1d4f7 Mon Sep 17 00:00:00 2001 From: Flo Date: Tue, 7 Oct 2025 11:46:13 +0200 Subject: [PATCH 7/7] use correct err msg --- go/apps/api/routes/v2_keys_update_key/handler.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go/apps/api/routes/v2_keys_update_key/handler.go b/go/apps/api/routes/v2_keys_update_key/handler.go index 430d8f4415..a7e512f863 100644 --- a/go/apps/api/routes/v2_keys_update_key/handler.go +++ b/go/apps/api/routes/v2_keys_update_key/handler.go @@ -619,8 +619,8 @@ func (h *Handler) Handle(ctx context.Context, s *zen.Session) error { if db.IsDuplicateKeyError(txErr) || db.IsDeadlockError(txErr) { return fault.Wrap(txErr, fault.Code(codes.App.Internal.ServiceUnavailable.URN()), - fault.Internal("failed to create key after retries"), - fault.Public("Failed to create key."), + fault.Internal("failed to update key after retries"), + fault.Public("Failed to update key."), ) } return txErr