Skip to content

feat: add external_id to order key #4379

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
chronark merged 18 commits into from
Nov 25, 2025
Merged

feat: add external_id to order key #4379

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
f9f3f0f
feat: add external_id to order key
chronark Nov 20, 2025
d05f220
revert: DateTime
chronark Nov 20, 2025
52e9c28
chore: update all table references
chronark Nov 20, 2025
9a1a409
chore: update TTLs
chronark Nov 20, 2025
85f1a16
Merge branch 'main' into 11-20-feat_add_external_id_to_order_key
chronark Nov 21, 2025
1b9da10
feat: update billing MVs to read from new tables
chronark Nov 21, 2025
544e15b
Merge branch '11-20-feat_add_external_id_to_order_key' of github.com:…
chronark Nov 21, 2025
ad4b83b
fix: migration script actually works decently fast
chronark Nov 21, 2025
f2a2028
[autofix.ci] apply automated fixes
autofix-ci[bot] Nov 21, 2025
fa1c549
wip
chronark Nov 24, 2025
0328828
fix: use higher granularity
chronark Nov 25, 2025
bdea519
Merge branch '11-20-feat_add_external_id_to_order_key' of github.com:…
chronark Nov 25, 2025
1f864b5
fix: delete cache
chronark Nov 25, 2025
d0e72e2
[autofix.ci] apply automated fixes
autofix-ci[bot] Nov 25, 2025
2131edc
docs: update table version
chronark Nov 25, 2025
c1d566c
Merge branch 'main' into 11-20-feat_add_external_id_to_order_key
chronark Nov 25, 2025
e68bc6a
Merge branch '11-20-feat_add_external_id_to_order_key' of github.com:…
chronark Nov 25, 2025
9268a89
fix: ttl
chronark Nov 25, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions apps/api/src/routes/v1_analytics_getVerifications.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -222,21 +222,21 @@ export const registerV1AnalyticsGetVerifications = (app: App) =>

const tables = {
hour: {
name: "default.key_verifications_per_hour_v2",
name: "default.key_verifications_per_hour_v3",
fill: `WITH FILL
FROM toStartOfHour(fromUnixTimestamp64Milli({ start: Int64 }))
TO toStartOfHour(fromUnixTimestamp64Milli({ end: Int64 }))
STEP INTERVAL 1 HOUR`,
},
day: {
name: "default.key_verifications_per_day_v2",
name: "default.key_verifications_per_day_v3",
fill: `WITH FILL
FROM toDate(toStartOfDay(fromUnixTimestamp64Milli({ start: Int64 })))
TO toDate(toStartOfDay(fromUnixTimestamp64Milli({ end: Int64 })))
STEP INTERVAL 1 DAY`,
},
month: {
name: "default.key_verifications_per_month_v2",
name: "default.key_verifications_per_month_v3",
fill: `WITH FILL
FROM toDate(toStartOfMonth(fromUnixTimestamp64Milli({ start: Int64 })))
TO toDate(toStartOfMonth(fromUnixTimestamp64Milli({ end: Int64 })))
Expand Down
8 changes: 4 additions & 4 deletions apps/engineering/content/docs/architecture/services/analytics.mdx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,10 +65,10 @@ Users query against friendly table names that map to actual ClickHouse tables:
```go
TableAliases: map[string]string{
"key_verifications_v1": "default.key_verifications_raw_v2",
"key_verifications_per_minute_v1": "default.key_verifications_per_minute_v2",
"key_verifications_per_hour_v1": "default.key_verifications_per_hour_v2",
"key_verifications_per_day_v1": "default.key_verifications_per_day_v2",
"key_verifications_per_month_v1": "default.key_verifications_per_month_v2",
"key_verifications_per_minute_v1": "default.key_verifications_per_minute_v3",
"key_verifications_per_hour_v1": "default.key_verifications_per_hour_v3",
"key_verifications_per_day_v1": "default.key_verifications_per_day_v3",
"key_verifications_per_month_v1": "default.key_verifications_per_month_v3",
}
```

Expand Down
16 changes: 8 additions & 8 deletions go/apps/api/routes/v2_analytics_get_verifications/handler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,18 +30,18 @@ type ResponseData = openapi.V2AnalyticsGetVerificationsResponseData
var (
tableAliases = map[string]string{
"key_verifications_v1": "default.key_verifications_raw_v2",
"key_verifications_per_minute_v1": "default.key_verifications_per_minute_v2",
"key_verifications_per_hour_v1": "default.key_verifications_per_hour_v2",
"key_verifications_per_day_v1": "default.key_verifications_per_day_v2",
"key_verifications_per_month_v1": "default.key_verifications_per_month_v2",
"key_verifications_per_minute_v1": "default.key_verifications_per_minute_v3",
"key_verifications_per_hour_v1": "default.key_verifications_per_hour_v3",
"key_verifications_per_day_v1": "default.key_verifications_per_day_v3",
"key_verifications_per_month_v1": "default.key_verifications_per_month_v3",
}

allowedTables = []string{
"default.key_verifications_raw_v2",
"default.key_verifications_per_minute_v2",
"default.key_verifications_per_hour_v2",
"default.key_verifications_per_day_v2",
"default.key_verifications_per_month_v2",
"default.key_verifications_per_minute_v3",
"default.key_verifications_per_hour_v3",
"default.key_verifications_per_day_v3",
"default.key_verifications_per_month_v3",
}
)

Expand Down
5 changes: 5 additions & 0 deletions go/cmd/create-clickhouse-user/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,11 +66,16 @@ unkey create-clickhouse-user --workspace-id ws_123 --username custom_user --max-

var allowedTables = []string{
// Key verifications
//
"default.key_verifications_raw_v2",
"default.key_verifications_per_minute_v2",
"default.key_verifications_per_minute_v3",
"default.key_verifications_per_hour_v2",
"default.key_verifications_per_hour_v3",
"default.key_verifications_per_day_v2",
"default.key_verifications_per_day_v3",
"default.key_verifications_per_month_v2",
"default.key_verifications_per_month_v3",
// Not used ATM
// // Ratelimits
// "default.ratelimits_raw_v2",
Expand Down
28 changes: 14 additions & 14 deletions go/pkg/clickhouse/key_verifications_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,7 @@ func TestKeyVerifications(t *testing.T) {
}, time.Minute, time.Second)

t.Run("totals are correct", func(t *testing.T) {
for _, table := range []string{"default.key_verifications_per_minute_v2", "default.key_verifications_per_hour_v2", "default.key_verifications_per_day_v2", "default.key_verifications_per_month_v2"} {
for _, table := range []string{"default.key_verifications_per_minute_v3", "default.key_verifications_per_hour_v3", "default.key_verifications_per_day_v3", "default.key_verifications_per_month_v3"} {
t.Run(table, func(t *testing.T) {
require.EventuallyWithT(t, func(c *assert.CollectT) {
queried := int64(0)
Expand All @@ -138,7 +138,7 @@ func TestKeyVerifications(t *testing.T) {
return acc
}, map[string]int{})

for _, table := range []string{"default.key_verifications_per_minute_v2", "default.key_verifications_per_hour_v2", "default.key_verifications_per_day_v2", "default.key_verifications_per_month_v2"} {
for _, table := range []string{"default.key_verifications_per_minute_v3", "default.key_verifications_per_hour_v3", "default.key_verifications_per_day_v3", "default.key_verifications_per_month_v3"} {
t.Run(table, func(t *testing.T) {
for outcome, count := range countByOutcome {
require.EventuallyWithT(t, func(c *assert.CollectT) {
Expand All @@ -164,7 +164,7 @@ func TestKeyVerifications(t *testing.T) {
return acc
}, map[string]int{})

for _, table := range []string{"default.key_verifications_per_minute_v2", "default.key_verifications_per_hour_v2", "default.key_verifications_per_day_v2", "default.key_verifications_per_month_v2"} {
for _, table := range []string{"default.key_verifications_per_minute_v3", "default.key_verifications_per_hour_v3", "default.key_verifications_per_day_v3", "default.key_verifications_per_month_v3"} {
t.Run(table, func(t *testing.T) {
t.Parallel()

Expand All @@ -191,7 +191,7 @@ func TestKeyVerifications(t *testing.T) {
return acc
}, map[string]int{})

for _, table := range []string{"default.key_verifications_per_minute_v2", "default.key_verifications_per_hour_v2", "default.key_verifications_per_day_v2", "default.key_verifications_per_month_v2"} {
for _, table := range []string{"default.key_verifications_per_minute_v3", "default.key_verifications_per_hour_v3", "default.key_verifications_per_day_v3", "default.key_verifications_per_month_v3"} {
t.Run(table, func(t *testing.T) {
for outcome, count := range countByOutcome {
require.EventuallyWithT(t, func(c *assert.CollectT) {
Expand Down Expand Up @@ -219,7 +219,7 @@ func TestKeyVerifications(t *testing.T) {
return acc
}, map[string]int{})

for _, table := range []string{"default.key_verifications_per_minute_v2", "default.key_verifications_per_hour_v2", "default.key_verifications_per_day_v2", "default.key_verifications_per_month_v2"} {
for _, table := range []string{"default.key_verifications_per_minute_v3", "default.key_verifications_per_hour_v3", "default.key_verifications_per_day_v3", "default.key_verifications_per_month_v3"} {
t.Run(table, func(t *testing.T) {
for outcome, count := range countByOutcome {
require.EventuallyWithT(t, func(c *assert.CollectT) {
Expand All @@ -242,7 +242,7 @@ func TestKeyVerifications(t *testing.T) {
p75 := percentile(latencies, 0.75)
p99 := percentile(latencies, 0.99)

for _, table := range []string{"default.key_verifications_per_minute_v2", "default.key_verifications_per_hour_v2", "default.key_verifications_per_day_v2", "default.key_verifications_per_month_v2"} {
for _, table := range []string{"default.key_verifications_per_minute_v3", "default.key_verifications_per_hour_v3", "default.key_verifications_per_day_v3", "default.key_verifications_per_month_v3"} {
t.Run(table, func(t *testing.T) {
t.Parallel()
var (
Expand All @@ -266,7 +266,7 @@ func TestKeyVerifications(t *testing.T) {
return acc + v.SpentCredits
}, int64(0))

for _, table := range []string{"default.key_verifications_per_minute_v2", "default.key_verifications_per_hour_v2", "default.key_verifications_per_day_v2", "default.key_verifications_per_month_v2"} {
for _, table := range []string{"default.key_verifications_per_minute_v3", "default.key_verifications_per_hour_v3", "default.key_verifications_per_day_v3", "default.key_verifications_per_month_v3"} {
t.Run(table, func(t *testing.T) {
t.Parallel()
var queried int64
Expand All @@ -287,7 +287,7 @@ func TestKeyVerifications(t *testing.T) {
return acc
}, int64(0))

for _, table := range []string{"default.key_verifications_per_minute_v2", "default.key_verifications_per_hour_v2", "default.key_verifications_per_day_v2", "default.key_verifications_per_month_v2"} {
for _, table := range []string{"default.key_verifications_per_minute_v3", "default.key_verifications_per_hour_v3", "default.key_verifications_per_day_v3", "default.key_verifications_per_month_v3"} {
t.Run(table, func(t *testing.T) {
t.Parallel()
var queried int64
Expand All @@ -311,7 +311,7 @@ func TestKeyVerifications(t *testing.T) {
return acc
}, int64(0))

for _, table := range []string{"default.key_verifications_per_minute_v2", "default.key_verifications_per_hour_v2", "default.key_verifications_per_day_v2", "default.key_verifications_per_month_v2"} {
for _, table := range []string{"default.key_verifications_per_minute_v3", "default.key_verifications_per_hour_v3", "default.key_verifications_per_day_v3", "default.key_verifications_per_month_v3"} {
t.Run(table, func(t *testing.T) {
t.Parallel()
var queried int64
Expand All @@ -332,7 +332,7 @@ func TestKeyVerifications(t *testing.T) {
id := identityID
expectedExternalID := identityToExternalID[id]

for _, table := range []string{"default.key_verifications_per_minute_v2", "default.key_verifications_per_hour_v2", "default.key_verifications_per_day_v2", "default.key_verifications_per_month_v2"} {
for _, table := range []string{"default.key_verifications_per_minute_v3", "default.key_verifications_per_hour_v3", "default.key_verifications_per_day_v3", "default.key_verifications_per_month_v3"} {
tbl := table
t.Run(tbl, func(t *testing.T) {
t.Parallel()
Expand Down Expand Up @@ -360,7 +360,7 @@ func TestKeyVerifications(t *testing.T) {
return acc
}, 0)

for _, table := range []string{"default.key_verifications_per_minute_v2", "default.key_verifications_per_hour_v2", "default.key_verifications_per_day_v2", "default.key_verifications_per_month_v2"} {
for _, table := range []string{"default.key_verifications_per_minute_v3", "default.key_verifications_per_hour_v3", "default.key_verifications_per_day_v3", "default.key_verifications_per_month_v3"} {
tbl := table
t.Run(tbl, func(t *testing.T) {
t.Parallel()
Expand Down Expand Up @@ -389,7 +389,7 @@ func TestKeyVerifications(t *testing.T) {
return acc
}, map[string]int{})

for _, table := range []string{"default.key_verifications_per_minute_v2", "default.key_verifications_per_hour_v2", "default.key_verifications_per_day_v2", "default.key_verifications_per_month_v2"} {
for _, table := range []string{"default.key_verifications_per_minute_v3", "default.key_verifications_per_hour_v3", "default.key_verifications_per_day_v3", "default.key_verifications_per_month_v3"} {
tbl := table
t.Run(tbl, func(t *testing.T) {
t.Parallel()
Expand All @@ -415,7 +415,7 @@ func TestKeyVerifications(t *testing.T) {
id := identityID
extID := identityToExternalID[id]

for _, table := range []string{"default.key_verifications_per_minute_v2", "default.key_verifications_per_hour_v2", "default.key_verifications_per_day_v2", "default.key_verifications_per_month_v2"} {
for _, table := range []string{"default.key_verifications_per_minute_v3", "default.key_verifications_per_hour_v3", "default.key_verifications_per_day_v3", "default.key_verifications_per_month_v3"} {
tbl := table
t.Run(tbl, func(t *testing.T) {
t.Parallel()
Expand Down Expand Up @@ -455,7 +455,7 @@ func TestKeyVerifications(t *testing.T) {
return acc
}, int64(0))

for _, table := range []string{"default.key_verifications_per_minute_v2", "default.key_verifications_per_hour_v2", "default.key_verifications_per_day_v2", "default.key_verifications_per_month_v2"} {
for _, table := range []string{"default.key_verifications_per_minute_v3", "default.key_verifications_per_hour_v3", "default.key_verifications_per_day_v3", "default.key_verifications_per_month_v3"} {
tbl := table
t.Run(tbl, func(t *testing.T) {
t.Parallel()
Expand Down
Loading
Loading