Skip to content

feat: bulk insert plugin for sqlc #3631

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
chronark merged 11 commits into from
Jul 18, 2025
Merged

feat: bulk insert plugin for sqlc #3631

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
29fc95d
Add sqlc plugin for bulk inserts
Flo4604 Jul 17, 2025
e89ab3c
Merge branch 'main' into sqlc_bulk_plugin
Flo4604 Jul 17, 2025
788455e
fix fmt
Flo4604 Jul 17, 2025
0547a14
Merge branch 'sqlc_bulk_plugin' of github.com:unkeyed/unkey into sqlc…
Flo4604 Jul 17, 2025
ed2f8b0
Merge branch 'main' into sqlc_bulk_plugin
Flo4604 Jul 17, 2025
bd6c192
remove wrong tests
Flo4604 Jul 17, 2025
4299206
Merge branch 'sqlc_bulk_plugin' of github.com:unkeyed/unkey into sqlc…
Flo4604 Jul 17, 2025
6453248
Merge branch 'main' into sqlc_bulk_plugin
Flo4604 Jul 17, 2025
632df1e
make it its own queries and pluralize function names
Flo4604 Jul 17, 2025
36039c5
Merge branch 'sqlc_bulk_plugin' of github.com:unkeyed/unkey into sqlc…
Flo4604 Jul 17, 2025
5b144d4
fix column name
Flo4604 Jul 17, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions go/apps/api/routes/v2_identities_create_identity/handler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -185,10 +185,9 @@ func (h *Handler) Handle(ctx context.Context, s *zen.Session) error {
})
}

err = db.BulkInsert(
err = db.BulkQuery.InsertIdentityRatelimits(
ctx,
tx,
"INSERT INTO ratelimits (id, workspace_id, identity_id, name, `limit`, duration, created_at, auto_apply) VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
rateLimitsToInsert,
)
if err != nil {
Expand Down
10 changes: 1 addition & 9 deletions go/apps/api/routes/v2_identities_update_identity/handler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -346,15 +346,7 @@ func (h *Handler) Handle(ctx context.Context, s *zen.Session) error {
}

if len(rateLimitsToInsert) > 0 {
err = db.BulkInsert(ctx, tx,
"INSERT INTO ratelimits (id, workspace_id, identity_id, name,`limit`"+`, duration, created_at, auto_apply) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
ON DUPLICATE KEY UPDATE`+"`limit` = VALUES(`limit`),"+`
name = VALUES(name),
duration = VALUES(duration),
auto_apply = VALUES(auto_apply),
updated_at = NOW()`,
rateLimitsToInsert,
)
err = db.BulkQuery.InsertIdentityRatelimits(ctx, tx, rateLimitsToInsert)

if err != nil {
// nolint:exhaustruct
Expand Down
22 changes: 4 additions & 18 deletions go/apps/api/routes/v2_keys_create_key/handler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -338,10 +338,7 @@ func (h *Handler) Handle(ctx context.Context, s *zen.Session) error {
}
}

err = db.BulkInsert(ctx, tx,
"INSERT INTO ratelimits (id, workspace_id, key_id, name, `limit`, duration, created_at, auto_apply) VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
ratelimitsToInsert,
)
err = db.BulkQuery.InsertKeyRatelimits(ctx, tx, ratelimitsToInsert)
if err != nil {
return fault.Wrap(err,
fault.Code(codes.App.Internal.ServiceUnavailable.URN()),
Expand Down Expand Up @@ -398,11 +395,7 @@ func (h *Handler) Handle(ctx context.Context, s *zen.Session) error {
}

if len(permissionsToCreate) > 0 {
err = db.BulkInsert(ctx, tx,
"INSERT INTO permissions (id, workspace_id, name, slug, description, created_at_m) VALUES (?, ?, ?, ?, ?, ?)",
permissionsToCreate,
)

err = db.BulkQuery.InsertPermissions(ctx, tx, permissionsToCreate)
if err != nil {
return fault.Wrap(err,
fault.Code(codes.App.Internal.ServiceUnavailable.URN()),
Expand Down Expand Up @@ -451,10 +444,7 @@ func (h *Handler) Handle(ctx context.Context, s *zen.Session) error {
}

if len(permissionsToInsert) > 0 {
err = db.BulkInsert(ctx, tx,
"INSERT INTO keys_permissions (key_id, permission_id, workspace_id, created_at_m) VALUES (?, ?, ?, ?)",
permissionsToInsert,
)
err = db.BulkQuery.InsertKeyPermissions(ctx, tx, permissionsToInsert)
if err != nil {
return fault.Wrap(err,
fault.Code(codes.App.Internal.ServiceUnavailable.URN()),
Expand Down Expand Up @@ -541,11 +531,7 @@ func (h *Handler) Handle(ctx context.Context, s *zen.Session) error {
}

if len(rolesToInsert) > 0 {
err = db.BulkInsert(ctx, tx,
"INSERT INTO keys_roles (key_id, role_id, workspace_id, created_at_m) VALUES (?, ?, ?, ?)",
rolesToInsert,
)

err = db.BulkQuery.InsertKeyRoles(ctx, tx, rolesToInsert)
if err != nil {
return fault.Wrap(err,
fault.Code(codes.App.Internal.ServiceUnavailable.URN()),
Expand Down
24 changes: 4 additions & 20 deletions go/apps/api/routes/v2_keys_update_key/handler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -362,12 +362,7 @@ func (h *Handler) Handle(ctx context.Context, s *zen.Session) error {
}

if len(ratelimitsToInsert) > 0 {
err = db.BulkInsert(ctx, tx,
"INSERT INTO ratelimits (id, workspace_id, key_id, name, `limit`, duration, created_at, auto_apply) VALUES (?, ?, ?, ?, ?, ?, ?, ?) "+
"ON DUPLICATE KEY UPDATE `limit` = VALUES(`limit`), duration = VALUES(duration), auto_apply = VALUES(auto_apply), updated_at = NOW()",
ratelimitsToInsert,
)

err = db.BulkQuery.InsertKeyRatelimits(ctx, tx, ratelimitsToInsert)
if err != nil {
return fault.Wrap(err,
fault.Code(codes.App.Internal.ServiceUnavailable.URN()),
Expand Down Expand Up @@ -424,11 +419,7 @@ func (h *Handler) Handle(ctx context.Context, s *zen.Session) error {
}

if len(permissionsToCreate) > 0 {
err = db.BulkInsert(ctx, tx,
"INSERT INTO permissions (id, workspace_id, name, slug, description, created_at_m) VALUES (?, ?, ?, ?, ?, ?)",
permissionsToCreate,
)

err = db.BulkQuery.InsertPermissions(ctx, tx, permissionsToCreate)
if err != nil {
return fault.Wrap(err,
fault.Code(codes.App.Internal.ServiceUnavailable.URN()),
Expand Down Expand Up @@ -458,10 +449,7 @@ func (h *Handler) Handle(ctx context.Context, s *zen.Session) error {
}

if len(permissionsToInsert) > 0 {
err = db.BulkInsert(ctx, tx,
"INSERT INTO keys_permissions (key_id, permission_id, workspace_id, created_at_m) VALUES (?, ?, ?, ?)",
permissionsToInsert,
)
err = db.BulkQuery.InsertKeyPermissions(ctx, tx, permissionsToInsert)
if err != nil {
return fault.Wrap(err,
fault.Code(codes.App.Internal.ServiceUnavailable.URN()),
Expand Down Expand Up @@ -526,11 +514,7 @@ func (h *Handler) Handle(ctx context.Context, s *zen.Session) error {
}

if len(rolesToInsert) > 0 {
err = db.BulkInsert(ctx, tx,
"INSERT INTO keys_roles (key_id, role_id, workspace_id, created_at_m) VALUES (?, ?, ?, ?)",
rolesToInsert,
)

err = db.BulkQuery.InsertKeyRoles(ctx, tx, rolesToInsert)
if err != nil {
return fault.Wrap(err,
fault.Code(codes.App.Internal.ServiceUnavailable.URN()),
Expand Down
1 change: 1 addition & 0 deletions go/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ require (
github.com/prometheus/client_golang v1.22.0
github.com/redis/go-redis/v9 v9.9.0
github.com/shirou/gopsutil/v4 v4.25.5
github.com/sqlc-dev/plugin-sdk-go v1.23.0
github.com/sqlc-dev/sqlc v1.28.0
github.com/stretchr/testify v1.10.0
github.com/unkeyed/unkey/go/deploy/pkg/tls v0.0.0-00010101000000-000000000000
Expand Down
2 changes: 2 additions & 0 deletions go/go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -291,6 +291,8 @@ github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An
github.com/spiffe/go-spiffe/v2 v2.5.0 h1:N2I01KCUkv1FAjZXJMwh95KK1ZIQLYbPfhaxw8WS0hE=
github.com/spiffe/go-spiffe/v2 v2.5.0/go.mod h1:P+NxobPc6wXhVtINNtFjNWGBTreew1GBUCwT2wPmb7g=
github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0=
github.com/sqlc-dev/plugin-sdk-go v1.23.0 h1:iSeJhnXPlbDXlbzUEebw/DxsGzE9rdDJArl8Hvt0RMM=
github.com/sqlc-dev/plugin-sdk-go v1.23.0/go.mod h1:I1r4THOfyETD+LI2gogN2LX8wCjwUZrgy/NU4In3llA=
github.com/sqlc-dev/sqlc v1.28.0 h1:2QB4X22pKNpKMyb8dRLnqZwMXW6S+ZCyYCpa+3/ICcI=
github.com/sqlc-dev/sqlc v1.28.0/go.mod h1:x6wDsOHH60dTX3ES9sUUxRVaROg5aFB3l3nkkjyuK1A=
github.com/stoewer/go-strcase v1.2.0 h1:Z2iHWqGXH00XYgqDmNgQbIBxf3wrNq0F3feEy0ainaU=
Expand Down
62 changes: 2 additions & 60 deletions go/internal/services/auditlogs/insert.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,64 +13,6 @@ import (
"github.com/unkeyed/unkey/go/pkg/uid"
)

// Generated Insert query which we are re-using for bulk inserts
const insertAuditLog = `INSERT INTO ` + "`" + `audit_log` + "`" + ` (
id,
workspace_id,
bucket_id,
bucket,
event,
time,
display,
remote_ip,
user_agent,
actor_type,
actor_id,
actor_name,
actor_meta,
created_at
) VALUES (
?,
?,
?,
?,
?,
?,
?,
?,
?,
?,
?,
?,
?,
?
)`

// Generated Insert query which we are re-using for bulk inserts
const insertAuditLogTarget = `INSERT INTO ` + "`" + `audit_log_target` + "`" + ` (
workspace_id,
bucket_id,
bucket,
audit_log_id,
display_name,
type,
id,
name,
meta,
created_at
) VALUES (
?,
?,
?,
?,
?,
?,
?,
?,
?,
?
)`

// DEFAULT_BUCKET is the default bucket name used for audit logs when no bucket
// is specified. All audit logs are categorized into buckets for organization
// and querying purposes, with "unkey_mutations" serving as the standard bucket
Expand Down Expand Up @@ -167,15 +109,15 @@ func (s *service) insertLogs(ctx context.Context, tx db.DBTX, logs []auditlog.Au
}
}

err := db.BulkInsert(ctx, tx, insertAuditLog, auditLogs)
err := db.BulkQuery.InsertAuditLogs(ctx, tx, auditLogs)
if err != nil {
return fault.Wrap(err,
fault.Code(codes.App.Internal.ServiceUnavailable.URN()),
fault.Internal("database failed to insert audit logs"), fault.Public("Failed to insert audit logs"),
)
}

err = db.BulkInsert(ctx, tx, insertAuditLogTarget, auditLogTargets)
err = db.BulkQuery.InsertAuditLogTargets(ctx, tx, auditLogTargets)
if err != nil {
return fault.Wrap(err,
fault.Code(codes.App.Internal.ServiceUnavailable.URN()),
Expand Down
44 changes: 44 additions & 0 deletions go/pkg/db/bulk_api_insert.sql.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

51 changes: 51 additions & 0 deletions go/pkg/db/bulk_audit_log_insert.sql.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

47 changes: 47 additions & 0 deletions go/pkg/db/bulk_audit_log_target_insert.sql.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading