From 81f3e04c086258eee0868c8e232d6b1b2e2ba0a8 Mon Sep 17 00:00:00 2001 From: chronark Date: Sat, 7 Mar 2026 11:34:23 +0100 Subject: [PATCH 01/24] fix: improve rollbacks This addresses a few things. The most obvious one is lowering the freshness of some frontline caches to make rollbacks faster. I also moved the entire deployment workflow to a virtual object, keyed by workspace id. The reason is that we had some potential races and this is the pragmatic fix for right now. I could've done it per app too, but I actually think limiting concurrency in the beginning is not a bad thing. afterall, we're paying for all of these resources. --- gen/proto/hydra/v1/deploy.pb.go | 2 +- gen/proto/hydra/v1/deploy_restate.pb.go | 58 ++++----- svc/ctrl/api/deployment_integration_test.go | 2 +- svc/ctrl/proto/hydra/v1/deploy.proto | 2 +- svc/ctrl/worker/deploy/cilium_policy.go | 2 +- svc/ctrl/worker/deploy/compensation.go | 8 +- svc/ctrl/worker/deploy/deploy_handler.go | 118 +++++++++--------- svc/ctrl/worker/deploy/deployment_step.go | 4 +- svc/ctrl/worker/deploy/promote_handler.go | 2 +- svc/ctrl/worker/deploy/rollback_handler.go | 21 ++-- .../scale_down_idle_preview_deployments.go | 2 +- svc/ctrl/worker/githubwebhook/handle_push.go | 4 +- .../worker/routing/assign_domains_handler.go | 2 +- svc/frontline/services/caches/caches.go | 4 +- .../db/src/schema/frontline_routes.ts | 4 +- 15 files changed, 119 insertions(+), 116 deletions(-) diff --git a/gen/proto/hydra/v1/deploy.pb.go b/gen/proto/hydra/v1/deploy.pb.go index ee7adc022c5..8a775ec7ce4 100644 --- a/gen/proto/hydra/v1/deploy.pb.go +++ b/gen/proto/hydra/v1/deploy.pb.go @@ -588,7 +588,7 @@ const file_hydra_v1_deploy_proto_rawDesc = "" + "\x06Deploy\x12\x17.hydra.v1.DeployRequest\x1a\x18.hydra.v1.DeployResponse\"\x00\x12C\n" + "\bRollback\x12\x19.hydra.v1.RollbackRequest\x1a\x1a.hydra.v1.RollbackResponse\"\x00\x12@\n" + "\aPromote\x12\x18.hydra.v1.PromoteRequest\x1a\x19.hydra.v1.PromoteResponse\"\x00\x12\x88\x01\n" + - "\x1fScaleDownIdlePreviewDeployments\x120.hydra.v1.ScaleDownIdlePreviewDeploymentsRequest\x1a1.hydra.v1.ScaleDownIdlePreviewDeploymentsResponse\"\x00\x1a\x04\x98\x80\x01\x02B\x91\x01\n" + + "\x1fScaleDownIdlePreviewDeployments\x120.hydra.v1.ScaleDownIdlePreviewDeploymentsRequest\x1a1.hydra.v1.ScaleDownIdlePreviewDeploymentsResponse\"\x00\x1a\x04\x98\x80\x01\x01B\x91\x01\n" + "\fcom.hydra.v1B\vDeployProtoP\x01Z3github.com/unkeyed/unkey/gen/proto/hydra/v1;hydrav1\xa2\x02\x03HXX\xaa\x02\bHydra.V1\xca\x02\bHydra\\V1\xe2\x02\x14Hydra\\V1\\GPBMetadata\xea\x02\tHydra::V1b\x06proto3" var ( diff --git a/gen/proto/hydra/v1/deploy_restate.pb.go b/gen/proto/hydra/v1/deploy_restate.pb.go index 53b42b7bf51..bef51bd43b6 100644 --- a/gen/proto/hydra/v1/deploy_restate.pb.go +++ b/gen/proto/hydra/v1/deploy_restate.pb.go @@ -52,16 +52,16 @@ type DeployServiceClient interface { } type deployServiceClient struct { - ctx sdk_go.Context - workflowID string - options []sdk_go.ClientOption + ctx sdk_go.Context + key string + options []sdk_go.ClientOption } -func NewDeployServiceClient(ctx sdk_go.Context, workflowID string, opts ...sdk_go.ClientOption) DeployServiceClient { +func NewDeployServiceClient(ctx sdk_go.Context, key string, opts ...sdk_go.ClientOption) DeployServiceClient { cOpts := append([]sdk_go.ClientOption{sdk_go.WithProtoJSON}, opts...) return &deployServiceClient{ ctx, - workflowID, + key, cOpts, } } @@ -70,7 +70,7 @@ func (c *deployServiceClient) Deploy(opts ...sdk_go.ClientOption) sdk_go.Client[ if len(opts) > 0 { cOpts = append(append([]sdk_go.ClientOption{}, cOpts...), opts...) } - return sdk_go.WithRequestType[*DeployRequest](sdk_go.Workflow[*DeployResponse](c.ctx, "hydra.v1.DeployService", c.workflowID, "Deploy", cOpts...)) + return sdk_go.WithRequestType[*DeployRequest](sdk_go.Object[*DeployResponse](c.ctx, "hydra.v1.DeployService", c.key, "Deploy", cOpts...)) } func (c *deployServiceClient) Rollback(opts ...sdk_go.ClientOption) sdk_go.Client[*RollbackRequest, *RollbackResponse] { @@ -78,7 +78,7 @@ func (c *deployServiceClient) Rollback(opts ...sdk_go.ClientOption) sdk_go.Clien if len(opts) > 0 { cOpts = append(append([]sdk_go.ClientOption{}, cOpts...), opts...) } - return sdk_go.WithRequestType[*RollbackRequest](sdk_go.Workflow[*RollbackResponse](c.ctx, "hydra.v1.DeployService", c.workflowID, "Rollback", cOpts...)) + return sdk_go.WithRequestType[*RollbackRequest](sdk_go.Object[*RollbackResponse](c.ctx, "hydra.v1.DeployService", c.key, "Rollback", cOpts...)) } func (c *deployServiceClient) Promote(opts ...sdk_go.ClientOption) sdk_go.Client[*PromoteRequest, *PromoteResponse] { @@ -86,7 +86,7 @@ func (c *deployServiceClient) Promote(opts ...sdk_go.ClientOption) sdk_go.Client if len(opts) > 0 { cOpts = append(append([]sdk_go.ClientOption{}, cOpts...), opts...) } - return sdk_go.WithRequestType[*PromoteRequest](sdk_go.Workflow[*PromoteResponse](c.ctx, "hydra.v1.DeployService", c.workflowID, "Promote", cOpts...)) + return sdk_go.WithRequestType[*PromoteRequest](sdk_go.Object[*PromoteResponse](c.ctx, "hydra.v1.DeployService", c.key, "Promote", cOpts...)) } func (c *deployServiceClient) ScaleDownIdlePreviewDeployments(opts ...sdk_go.ClientOption) sdk_go.Client[*ScaleDownIdlePreviewDeploymentsRequest, *ScaleDownIdlePreviewDeploymentsResponse] { @@ -94,7 +94,7 @@ func (c *deployServiceClient) ScaleDownIdlePreviewDeployments(opts ...sdk_go.Cli if len(opts) > 0 { cOpts = append(append([]sdk_go.ClientOption{}, cOpts...), opts...) } - return sdk_go.WithRequestType[*ScaleDownIdlePreviewDeploymentsRequest](sdk_go.Workflow[*ScaleDownIdlePreviewDeploymentsResponse](c.ctx, "hydra.v1.DeployService", c.workflowID, "ScaleDownIdlePreviewDeployments", cOpts...)) + return sdk_go.WithRequestType[*ScaleDownIdlePreviewDeploymentsRequest](sdk_go.Object[*ScaleDownIdlePreviewDeploymentsResponse](c.ctx, "hydra.v1.DeployService", c.key, "ScaleDownIdlePreviewDeployments", cOpts...)) } // DeployServiceIngressClient is the ingress client API for hydra.v1.DeployService service. @@ -125,35 +125,35 @@ type DeployServiceIngressClient interface { type deployServiceIngressClient struct { client *ingress.Client serviceName string - workflowID string + key string } -func NewDeployServiceIngressClient(client *ingress.Client, workflowID string) DeployServiceIngressClient { +func NewDeployServiceIngressClient(client *ingress.Client, key string) DeployServiceIngressClient { return &deployServiceIngressClient{ client, "hydra.v1.DeployService", - workflowID, + key, } } func (c *deployServiceIngressClient) Deploy() ingress.Requester[*DeployRequest, *DeployResponse] { codec := encoding.ProtoJSONCodec - return ingress.NewRequester[*DeployRequest, *DeployResponse](c.client, c.serviceName, "Deploy", &c.workflowID, &codec) + return ingress.NewRequester[*DeployRequest, *DeployResponse](c.client, c.serviceName, "Deploy", &c.key, &codec) } func (c *deployServiceIngressClient) Rollback() ingress.Requester[*RollbackRequest, *RollbackResponse] { codec := encoding.ProtoJSONCodec - return ingress.NewRequester[*RollbackRequest, *RollbackResponse](c.client, c.serviceName, "Rollback", &c.workflowID, &codec) + return ingress.NewRequester[*RollbackRequest, *RollbackResponse](c.client, c.serviceName, "Rollback", &c.key, &codec) } func (c *deployServiceIngressClient) Promote() ingress.Requester[*PromoteRequest, *PromoteResponse] { codec := encoding.ProtoJSONCodec - return ingress.NewRequester[*PromoteRequest, *PromoteResponse](c.client, c.serviceName, "Promote", &c.workflowID, &codec) + return ingress.NewRequester[*PromoteRequest, *PromoteResponse](c.client, c.serviceName, "Promote", &c.key, &codec) } func (c *deployServiceIngressClient) ScaleDownIdlePreviewDeployments() ingress.Requester[*ScaleDownIdlePreviewDeploymentsRequest, *ScaleDownIdlePreviewDeploymentsResponse] { codec := encoding.ProtoJSONCodec - return ingress.NewRequester[*ScaleDownIdlePreviewDeploymentsRequest, *ScaleDownIdlePreviewDeploymentsResponse](c.client, c.serviceName, "ScaleDownIdlePreviewDeployments", &c.workflowID, &codec) + return ingress.NewRequester[*ScaleDownIdlePreviewDeploymentsRequest, *ScaleDownIdlePreviewDeploymentsResponse](c.client, c.serviceName, "ScaleDownIdlePreviewDeployments", &c.key, &codec) } // DeployServiceServer is the server API for hydra.v1.DeployService service. @@ -179,21 +179,21 @@ type DeployServiceServer interface { // containers across regions, wait for health, configure domain routing, and // update the project's live deployment pointer for production environments. // Sets deployment status to failed on any error. - Deploy(ctx sdk_go.WorkflowSharedContext, req *DeployRequest) (*DeployResponse, error) + Deploy(ctx sdk_go.ObjectContext, req *DeployRequest) (*DeployResponse, error) // Rollback switches sticky frontline routes (environment and live) from the // current live deployment back to a previous one. Marks the project as rolled // back so future deploys don't automatically reclaim live routes. // Source must be the current live deployment; both must share the same project // and environment. - Rollback(ctx sdk_go.WorkflowSharedContext, req *RollbackRequest) (*RollbackResponse, error) + Rollback(ctx sdk_go.ObjectContext, req *RollbackRequest) (*RollbackResponse, error) // Promote reassigns sticky frontline routes to a target deployment and clears // the rolled-back flag, restoring normal deployment flow. // Target must be in ready status and not already the live deployment. - Promote(ctx sdk_go.WorkflowSharedContext, req *PromoteRequest) (*PromoteResponse, error) + Promote(ctx sdk_go.ObjectContext, req *PromoteRequest) (*PromoteResponse, error) // ScaleDownIdlePreviewDeployments iterates all preview environments and sets any // deployment to standby that has received zero requests in the last 6 hours. // Intended to be called by a cron job. - ScaleDownIdlePreviewDeployments(ctx sdk_go.WorkflowSharedContext, req *ScaleDownIdlePreviewDeploymentsRequest) (*ScaleDownIdlePreviewDeploymentsResponse, error) + ScaleDownIdlePreviewDeployments(ctx sdk_go.ObjectContext, req *ScaleDownIdlePreviewDeploymentsRequest) (*ScaleDownIdlePreviewDeploymentsResponse, error) } // UnimplementedDeployServiceServer should be embedded to have @@ -203,16 +203,16 @@ type DeployServiceServer interface { // pointer dereference when methods are called. type UnimplementedDeployServiceServer struct{} -func (UnimplementedDeployServiceServer) Deploy(ctx sdk_go.WorkflowSharedContext, req *DeployRequest) (*DeployResponse, error) { +func (UnimplementedDeployServiceServer) Deploy(ctx sdk_go.ObjectContext, req *DeployRequest) (*DeployResponse, error) { return nil, sdk_go.TerminalError(fmt.Errorf("method Deploy not implemented"), 501) } -func (UnimplementedDeployServiceServer) Rollback(ctx sdk_go.WorkflowSharedContext, req *RollbackRequest) (*RollbackResponse, error) { +func (UnimplementedDeployServiceServer) Rollback(ctx sdk_go.ObjectContext, req *RollbackRequest) (*RollbackResponse, error) { return nil, sdk_go.TerminalError(fmt.Errorf("method Rollback not implemented"), 501) } -func (UnimplementedDeployServiceServer) Promote(ctx sdk_go.WorkflowSharedContext, req *PromoteRequest) (*PromoteResponse, error) { +func (UnimplementedDeployServiceServer) Promote(ctx sdk_go.ObjectContext, req *PromoteRequest) (*PromoteResponse, error) { return nil, sdk_go.TerminalError(fmt.Errorf("method Promote not implemented"), 501) } -func (UnimplementedDeployServiceServer) ScaleDownIdlePreviewDeployments(ctx sdk_go.WorkflowSharedContext, req *ScaleDownIdlePreviewDeploymentsRequest) (*ScaleDownIdlePreviewDeploymentsResponse, error) { +func (UnimplementedDeployServiceServer) ScaleDownIdlePreviewDeployments(ctx sdk_go.ObjectContext, req *ScaleDownIdlePreviewDeploymentsRequest) (*ScaleDownIdlePreviewDeploymentsResponse, error) { return nil, sdk_go.TerminalError(fmt.Errorf("method ScaleDownIdlePreviewDeployments not implemented"), 501) } func (UnimplementedDeployServiceServer) testEmbeddedByValue() {} @@ -233,10 +233,10 @@ func NewDeployServiceServer(srv DeployServiceServer, opts ...sdk_go.ServiceDefin t.testEmbeddedByValue() } sOpts := append([]sdk_go.ServiceDefinitionOption{sdk_go.WithProtoJSON}, opts...) - router := sdk_go.NewWorkflow("hydra.v1.DeployService", sOpts...) - router = router.Handler("Deploy", sdk_go.NewWorkflowSharedHandler(srv.Deploy)) - router = router.Handler("Rollback", sdk_go.NewWorkflowSharedHandler(srv.Rollback)) - router = router.Handler("Promote", sdk_go.NewWorkflowSharedHandler(srv.Promote)) - router = router.Handler("ScaleDownIdlePreviewDeployments", sdk_go.NewWorkflowSharedHandler(srv.ScaleDownIdlePreviewDeployments)) + router := sdk_go.NewObject("hydra.v1.DeployService", sOpts...) + router = router.Handler("Deploy", sdk_go.NewObjectHandler(srv.Deploy)) + router = router.Handler("Rollback", sdk_go.NewObjectHandler(srv.Rollback)) + router = router.Handler("Promote", sdk_go.NewObjectHandler(srv.Promote)) + router = router.Handler("ScaleDownIdlePreviewDeployments", sdk_go.NewObjectHandler(srv.ScaleDownIdlePreviewDeployments)) return router } diff --git a/svc/ctrl/api/deployment_integration_test.go b/svc/ctrl/api/deployment_integration_test.go index dd7777a224a..7804231306a 100644 --- a/svc/ctrl/api/deployment_integration_test.go +++ b/svc/ctrl/api/deployment_integration_test.go @@ -20,7 +20,7 @@ type mockDeployService struct { requests chan *hydrav1.DeployRequest } -func (m *mockDeployService) Deploy(ctx restate.WorkflowSharedContext, req *hydrav1.DeployRequest) (*hydrav1.DeployResponse, error) { +func (m *mockDeployService) Deploy(ctx restate.ObjectContext, req *hydrav1.DeployRequest) (*hydrav1.DeployResponse, error) { m.requests <- req return &hydrav1.DeployResponse{}, nil } diff --git a/svc/ctrl/proto/hydra/v1/deploy.proto b/svc/ctrl/proto/hydra/v1/deploy.proto index 437a0e7878c..ccb804d07b8 100644 --- a/svc/ctrl/proto/hydra/v1/deploy.proto +++ b/svc/ctrl/proto/hydra/v1/deploy.proto @@ -20,7 +20,7 @@ option go_package = "github.com/unkeyed/unkey/gen/proto/hydra/v1;hydrav1"; // targets a single deployment, typically sent with a 30-minute delay after a new // deployment replaces it. service DeployService { - option (dev.restate.sdk.go.service_type) = WORKFLOW; + option (dev.restate.sdk.go.service_type) = VIRTUAL_OBJECT; // Deploy executes the full deployment workflow: build (if git source), provision // containers across regions, wait for health, configure domain routing, and diff --git a/svc/ctrl/worker/deploy/cilium_policy.go b/svc/ctrl/worker/deploy/cilium_policy.go index 6b14e0eec90..af9e630ff28 100644 --- a/svc/ctrl/worker/deploy/cilium_policy.go +++ b/svc/ctrl/worker/deploy/cilium_policy.go @@ -37,7 +37,7 @@ type ciliumPolicySpec struct { // its own policies so that old deployments (serving traffic via SHA URLs) are not broken // when a new deployment changes the port. func (w *Workflow) ensureCiliumNetworkPolicy( - ctx restate.WorkflowSharedContext, + ctx restate.ObjectContext, workspace db.Workspace, project db.Project, environment db.Environment, diff --git a/svc/ctrl/worker/deploy/compensation.go b/svc/ctrl/worker/deploy/compensation.go index 70654e6eb98..5e5f4934ca8 100644 --- a/svc/ctrl/worker/deploy/compensation.go +++ b/svc/ctrl/worker/deploy/compensation.go @@ -16,14 +16,14 @@ import ( // setup sequence that succeeded before a failure. type Compensation struct { mu sync.Mutex - operations []func(ctx restate.WorkflowSharedContext) error + operations []func(ctx restate.ObjectContext) error } // NewCompensation creates an empty [Compensation]. func NewCompensation() *Compensation { return &Compensation{ mu: sync.Mutex{}, - operations: []func(ctx restate.WorkflowSharedContext) error{}, + operations: []func(ctx restate.ObjectContext) error{}, } } @@ -34,7 +34,7 @@ func NewCompensation() *Compensation { func (c *Compensation) Add(name string, run func(ctx restate.RunContext) error) { c.mu.Lock() defer c.mu.Unlock() - c.operations = append(c.operations, func(ctx restate.WorkflowSharedContext) error { + c.operations = append(c.operations, func(ctx restate.ObjectContext) error { return restate.RunVoid(ctx, run, restate.WithName(fmt.Sprintf("[compensation]: %s", name))) }) } @@ -44,7 +44,7 @@ func (c *Compensation) Add(name string, run func(ctx restate.RunContext) error) // Execute returns nil when every step succeeds. When multiple steps fail, it // joins all failures with [errors.Join]. Execute does not clear registered // steps, so calling it more than once re-runs the same compensations. -func (c *Compensation) Execute(ctx restate.WorkflowSharedContext) error { +func (c *Compensation) Execute(ctx restate.ObjectContext) error { c.mu.Lock() defer c.mu.Unlock() var errs error diff --git a/svc/ctrl/worker/deploy/deploy_handler.go b/svc/ctrl/worker/deploy/deploy_handler.go index 283e9104bdc..dccd6fe319c 100644 --- a/svc/ctrl/worker/deploy/deploy_handler.go +++ b/svc/ctrl/worker/deploy/deploy_handler.go @@ -52,7 +52,7 @@ const ( // // Returns terminal errors for validation failures and retryable errors for // transient system failures. -func (w *Workflow) Deploy(ctx restate.WorkflowSharedContext, req *hydrav1.DeployRequest) (_ *hydrav1.DeployResponse, retErr error) { +func (w *Workflow) Deploy(ctx restate.ObjectContext, req *hydrav1.DeployRequest) (_ *hydrav1.DeployResponse, retErr error) { err := assert.All( assert.NotEmpty(req.GetDeploymentId(), "deployment_id is required"), @@ -112,7 +112,7 @@ func (w *Workflow) Deploy(ctx restate.WorkflowSharedContext, req *hydrav1.Deploy ) // --- Starting --- - err = w.DeploymentStep(ctx, db.DeploymentStepsStepStarting, deployment, func(stepCtx restate.WorkflowSharedContext) error { + err = w.DeploymentStep(ctx, db.DeploymentStepsStepStarting, deployment, func(stepCtx restate.ObjectContext) error { workspace, err = restate.Run(ctx, func(runCtx restate.RunContext) (db.Workspace, error) { var ws db.Workspace @@ -176,7 +176,7 @@ func (w *Workflow) Deploy(ctx restate.WorkflowSharedContext, req *hydrav1.Deploy } // --- Build --- - err = w.DeploymentStep(ctx, db.DeploymentStepsStepBuilding, deployment, func(stepCtx restate.WorkflowSharedContext) error { + err = w.DeploymentStep(ctx, db.DeploymentStepsStepBuilding, deployment, func(stepCtx restate.ObjectContext) error { return w.buildImage(stepCtx, req, &deployment) }) if err != nil { @@ -184,7 +184,7 @@ func (w *Workflow) Deploy(ctx restate.WorkflowSharedContext, req *hydrav1.Deploy } // --- Deploy --- - err = w.DeploymentStep(ctx, db.DeploymentStepsStepDeploying, deployment, func(stepCtx restate.WorkflowSharedContext) error { + err = w.DeploymentStep(ctx, db.DeploymentStepsStepDeploying, deployment, func(stepCtx restate.ObjectContext) error { topologies, err := w.createTopologies(stepCtx, compensation, workspace, deployment) if err != nil { @@ -210,7 +210,7 @@ func (w *Workflow) Deploy(ctx restate.WorkflowSharedContext, req *hydrav1.Deploy } // --- Network --- - err = w.DeploymentStep(ctx, db.DeploymentStepsStepNetwork, deployment, func(stepCtx restate.WorkflowSharedContext) error { + err = w.DeploymentStep(ctx, db.DeploymentStepsStepNetwork, deployment, func(stepCtx restate.ObjectContext) error { return w.configureRouting(stepCtx, workspace, project, app, environment, deployment) }) @@ -219,7 +219,7 @@ func (w *Workflow) Deploy(ctx restate.WorkflowSharedContext, req *hydrav1.Deploy } // --- Finalize --- - err = w.DeploymentStep(ctx, db.DeploymentStepsStepFinalizing, deployment, func(stepCtx restate.WorkflowSharedContext) error { + err = w.DeploymentStep(ctx, db.DeploymentStepsStepFinalizing, deployment, func(stepCtx restate.ObjectContext) error { err = restate.RunVoid(ctx, func(stepCtx restate.RunContext) error { return db.Query.UpdateDeploymentStatus(stepCtx, w.db.RW(), db.UpdateDeploymentStatusParams{ ID: deployment.ID, @@ -259,7 +259,7 @@ func (w *Workflow) Deploy(ctx restate.WorkflowSharedContext, req *hydrav1.Deploy // // Returns a terminal error for unknown source types and build failures that // cannot be retried (e.g. bad Dockerfile). -func (w *Workflow) buildImage(ctx restate.WorkflowSharedContext, req *hydrav1.DeployRequest, deployment *db.Deployment) error { +func (w *Workflow) buildImage(ctx restate.ObjectContext, req *hydrav1.DeployRequest, deployment *db.Deployment) error { dockerImage := "" switch source := req.GetSource().(type) { @@ -381,7 +381,7 @@ func (w *Workflow) buildImage(ctx restate.WorkflowSharedContext, req *hydrav1.De // topology. Compensation deletes by deployment, region, and version so retries // never remove topologies created by a newer attempt. func (w *Workflow) createTopologies( - ctx restate.WorkflowSharedContext, + ctx restate.ObjectContext, compensation *Compensation, workspace db.Workspace, deployment db.Deployment, @@ -485,7 +485,7 @@ func (w *Workflow) createTopologies( // idempotent — if a concurrent workflow already created the sentinel, the // duplicate key error is silently ignored. func (w *Workflow) ensureSentinels( - ctx restate.WorkflowSharedContext, + ctx restate.ObjectContext, workspace db.Workspace, project db.Project, environment db.Environment, @@ -576,7 +576,7 @@ func (w *Workflow) ensureSentinels( // [hydrav1.AssignFrontlineRoutesRequest] so that the routing layer atomically // switches traffic to this deployment's topologies. func (w *Workflow) configureRouting( - ctx restate.WorkflowSharedContext, + ctx restate.ObjectContext, workspace db.Workspace, project db.Project, app db.App, @@ -630,39 +630,49 @@ func (w *Workflow) configureRouting( } } - // Derive rolled-back state: if the current live deployment doesn't match - // the previous latest-ready deployment, a rollback occurred and we should - // not auto-promote this new deployment. - autoPromote := false - if environment.Slug == "production" { - if !app.CurrentDeploymentID.Valid { - // No current deployment yet — first deploy, auto-promote - autoPromote = true - } else { - prevLatest, prevErr := restate.Run(ctx, func(runCtx restate.RunContext) (string, error) { - return db.Query.FindLatestReadyDeploymentByAppAndEnv(runCtx, w.db.RO(), db.FindLatestReadyDeploymentByAppAndEnvParams{ - AppID: app.ID, - EnvironmentID: deployment.EnvironmentID, - ExcludeID: deployment.ID, - }) - }, restate.WithName("check rolled back state")) - if prevErr == nil && prevLatest == app.CurrentDeploymentID.String { - autoPromote = true - } - } + // refresh app, cause it might have changed since we read it at the beginning of the workflow (e.g. another deployment promoted to live and updated current_deployment_id) + app, err := restate.Run(ctx, func(runCtx restate.RunContext) (db.App, error) { + return db.Query.FindAppById(runCtx, w.db.RO(), app.ID) + }) + if err != nil { + return fault.Wrap(err, fault.Public("Failed to read from database. Please try again.")) } - // Fetch sticky routes for this environment - stickyTypes := []db.FrontlineRoutesSticky{db.FrontlineRoutesStickyEnvironment} - if autoPromote { - stickyTypes = append(stickyTypes, db.FrontlineRoutesStickyLive) - } + routeIDs, err := restate.Run(ctx, func(runCtx restate.RunContext) ([]string, error) { - stickyRoutes, err := restate.Run(ctx, func(stepCtx restate.RunContext) ([]db.FindFrontlineRouteForPromotionRow, error) { - return db.Query.FindFrontlineRouteForPromotion(stepCtx, w.db.RO(), db.FindFrontlineRouteForPromotionParams{ - EnvironmentID: deployment.EnvironmentID, - Sticky: stickyTypes, + // using a transaction here to ensure we read a consistent set of sticky routes that won't change under us as we promote this deployment. + // This is important to prevent a race + return db.TxWithResult(runCtx, w.db.RW(), func(txCtx context.Context, tx db.DBTX) ([]string, error) { + // Fetch sticky routes for this environment + stickyTypes := []db.FrontlineRoutesSticky{db.FrontlineRoutesStickyEnvironment} + if !app.IsRolledBack { + stickyTypes = append(stickyTypes, db.FrontlineRoutesStickyLive) + } + app, err := db.Query.FindAppById(txCtx, tx, app.ID) + if err != nil { + return nil, err + } + + // if the app is rolled back, we should not consider live sticky routes for promotion, even for production, because the live deployment is not healthy and should not receive traffic + if app.IsRolledBack { + stickyTypes = []db.FrontlineRoutesSticky{db.FrontlineRoutesStickyEnvironment} + } + + routes, err := db.Query.FindFrontlineRouteForPromotion(txCtx, tx, db.FindFrontlineRouteForPromotionParams{ + EnvironmentID: deployment.EnvironmentID, + Sticky: stickyTypes, + }) + if err != nil { + return nil, err + } + + routeIDs := make([]string, len(routes)) + for i, route := range routes { + routeIDs[i] = route.ID + } + return routeIDs, nil }) + }, restate.WithName("finding sticky routes")) if err != nil { return fault.Wrap( @@ -671,15 +681,11 @@ func (w *Workflow) configureRouting( ) } - for _, route := range stickyRoutes { - existingRouteIDs = append(existingRouteIDs, route.ID) - } - // Key routing service by app ID for per-app serialization _, err = hydrav1.NewRoutingServiceClient(ctx, app.ID). AssignFrontlineRoutes().Request(&hydrav1.AssignFrontlineRoutesRequest{ DeploymentId: deployment.ID, - FrontlineRouteIds: existingRouteIDs, + FrontlineRouteIds: routeIDs, }) if err != nil { return fault.Wrap( @@ -691,7 +697,7 @@ func (w *Workflow) configureRouting( return nil } -// swapLiveDeployment atomically updates the project's live deployment pointer to +// swapLiveDeployment atomically updates the apps's live deployment pointer to // this deployment and schedules the previous live deployment for standby after // 30 minutes via [hydrav1.DeploymentServiceClient.ScheduleDesiredStateChange]. // @@ -702,7 +708,7 @@ func (w *Workflow) configureRouting( // This only applies to production environments that are not in a rolled-back state; // for all other cases the method is a no-op and returns nil. func (w *Workflow) swapLiveDeployment( - ctx restate.WorkflowSharedContext, + ctx restate.ObjectContext, deployment db.Deployment, app db.App, environment db.Environment, @@ -711,10 +717,10 @@ func (w *Workflow) swapLiveDeployment( return nil } - // Atomically read the current live deployment and swap it to the new one. + // Atomically read the current deployment and swap it to the new one. // This prevents a race where two concurrent deploys both capture the same - // previousLiveDeploymentID and one of them never gets scheduled for standby. - previousLiveDeploymentID, err := restate.Run(ctx, func(runCtx restate.RunContext) (sql.NullString, error) { + // currentDeploymentID and one of them never gets scheduled for standby. + previousDeploymentID, err := restate.Run(ctx, func(runCtx restate.RunContext) (sql.NullString, error) { return db.TxWithResult(runCtx, w.db.RW(), func(txCtx context.Context, tx db.DBTX) (sql.NullString, error) { currentApp, findErr := db.Query.FindAppById(txCtx, tx, deployment.AppID) if findErr != nil { @@ -723,7 +729,7 @@ func (w *Workflow) swapLiveDeployment( updateErr := db.Query.UpdateAppDeployments(txCtx, tx, db.UpdateAppDeploymentsParams{ IsRolledBack: false, - AppID: currentApp.ID, + AppID: deployment.AppID, CurrentDeploymentID: sql.NullString{Valid: true, String: deployment.ID}, UpdatedAt: sql.NullInt64{Valid: true, Int64: time.Now().UnixMilli()}, }) @@ -733,19 +739,19 @@ func (w *Workflow) swapLiveDeployment( return currentApp.CurrentDeploymentID, nil }) - }, restate.WithName("swapping project live deployment")) + }, restate.WithName("swapping app live deployment")) if err != nil { - return fault.Wrap(err, fault.Public("Project live deployment could not be updated.")) + return fault.Wrap(err, fault.Public("App live deployment could not be updated.")) } - if previousLiveDeploymentID.Valid { - _, err = hydrav1.NewDeploymentServiceClient(ctx, previousLiveDeploymentID.String). + if previousDeploymentID.Valid { + _, err = hydrav1.NewDeploymentServiceClient(ctx, previousDeploymentID.String). ScheduleDesiredStateChange().Request( &hydrav1.ScheduleDesiredStateChangeRequest{ DelayMillis: (30 * time.Minute).Milliseconds(), State: hydrav1.DeploymentDesiredState_DEPLOYMENT_DESIRED_STATE_STANDBY, }, - restate.WithIdempotencyKey(deployment.ID), + restate.WithIdempotencyKey(previousDeploymentID.String), ) if err != nil { return fault.Wrap(err, fault.Public("Previous live deployment could not be scheduled for standby.")) @@ -764,7 +770,7 @@ func (w *Workflow) swapLiveDeployment( // workflow proceeds to routing, while still allowing single-region deployments // to pass. Regions that time out or error are skipped rather than failing the // entire deployment, so a degraded region does not block progress. -func (w *Workflow) waitForDeployments(ctx restate.WorkflowSharedContext, deploymentID string, topologies []db.InsertDeploymentTopologyParams) error { +func (w *Workflow) waitForDeployments(ctx restate.ObjectContext, deploymentID string, topologies []db.InsertDeploymentTopologyParams) error { logger.Info("waiting for deployments to be ready", "deployment_id", deploymentID) deadline, err := restate.Run(ctx, func(_ restate.RunContext) (time.Time, error) { diff --git a/svc/ctrl/worker/deploy/deployment_step.go b/svc/ctrl/worker/deploy/deployment_step.go index 430105aa84e..909799aca2c 100644 --- a/svc/ctrl/worker/deploy/deployment_step.go +++ b/svc/ctrl/worker/deploy/deployment_step.go @@ -12,10 +12,10 @@ import ( ) func (w *Workflow) DeploymentStep( - ctx restate.WorkflowSharedContext, + ctx restate.ObjectContext, step db.DeploymentStepsStep, deployment db.Deployment, - fn func(innerCtx restate.WorkflowSharedContext) error) error { + fn func(innerCtx restate.ObjectContext) error) error { err := restate.RunVoid(ctx, func(runCtx restate.RunContext) error { now := time.Now().UnixMilli() diff --git a/svc/ctrl/worker/deploy/promote_handler.go b/svc/ctrl/worker/deploy/promote_handler.go index 08786beeee9..9a56f34091a 100644 --- a/svc/ctrl/worker/deploy/promote_handler.go +++ b/svc/ctrl/worker/deploy/promote_handler.go @@ -30,7 +30,7 @@ import ( // // Returns terminal errors (400/404) for validation failures and retryable errors // for system failures. -func (w *Workflow) Promote(ctx restate.WorkflowSharedContext, req *hydrav1.PromoteRequest) (*hydrav1.PromoteResponse, error) { +func (w *Workflow) Promote(ctx restate.ObjectContext, req *hydrav1.PromoteRequest) (*hydrav1.PromoteResponse, error) { logger.Info("initiating promotion", "target", req.GetTargetDeploymentId()) // Get target deployment diff --git a/svc/ctrl/worker/deploy/rollback_handler.go b/svc/ctrl/worker/deploy/rollback_handler.go index f4ecd34378f..d040e62a281 100644 --- a/svc/ctrl/worker/deploy/rollback_handler.go +++ b/svc/ctrl/worker/deploy/rollback_handler.go @@ -7,6 +7,7 @@ import ( restate "github.com/restatedev/sdk-go" hydrav1 "github.com/unkeyed/unkey/gen/proto/hydra/v1" + "github.com/unkeyed/unkey/pkg/assert" "github.com/unkeyed/unkey/pkg/db" "github.com/unkeyed/unkey/pkg/logger" ) @@ -30,7 +31,7 @@ import ( // // Returns terminal errors (400/404) for validation failures and retryable errors // for system failures. -func (w *Workflow) Rollback(ctx restate.WorkflowSharedContext, req *hydrav1.RollbackRequest) (*hydrav1.RollbackResponse, error) { +func (w *Workflow) Rollback(ctx restate.ObjectContext, req *hydrav1.RollbackRequest) (*hydrav1.RollbackResponse, error) { logger.Info("initiating rollback", "source", req.GetSourceDeploymentId(), "target", req.GetTargetDeploymentId(), @@ -63,17 +64,13 @@ func (w *Workflow) Rollback(ctx restate.WorkflowSharedContext, req *hydrav1.Roll return nil, fmt.Errorf("failed to get target deployment: %w", err) } - // Validate deployments are in same environment and project - if targetDeployment.EnvironmentID != sourceDeployment.EnvironmentID { - return nil, restate.TerminalError(fmt.Errorf("deployments must be in the same environment"), 400) - } - - if targetDeployment.ProjectID != sourceDeployment.ProjectID { - return nil, restate.TerminalError(fmt.Errorf("deployments must be in the same project"), 400) - } - - if targetDeployment.AppID != sourceDeployment.AppID { - return nil, restate.TerminalError(fmt.Errorf("deployments must be in the same app"), 400) + err = assert.All( + assert.Equal(targetDeployment.ProjectID, sourceDeployment.ProjectID, "deployments must be in the same project"), + assert.Equal(targetDeployment.AppID, sourceDeployment.AppID, "deployments must be in the same app"), + assert.Equal(targetDeployment.EnvironmentID, sourceDeployment.EnvironmentID, "deployments must be in the same environment"), + ) + if err != nil { + return nil, restate.TerminalError(err, 400) } // Get app from deployment's app_id diff --git a/svc/ctrl/worker/deploy/scale_down_idle_preview_deployments.go b/svc/ctrl/worker/deploy/scale_down_idle_preview_deployments.go index bc0f07510ea..28625ff3a1f 100644 --- a/svc/ctrl/worker/deploy/scale_down_idle_preview_deployments.go +++ b/svc/ctrl/worker/deploy/scale_down_idle_preview_deployments.go @@ -20,7 +20,7 @@ var idleTime = 6 * time.Hour // branches that are no longer actively used, so this workflow paginates through // all preview environments and transitions idle deployments to archived by // checking request counts in ClickHouse. -func (w *Workflow) ScaleDownIdlePreviewDeployments(ctx restate.WorkflowSharedContext, req *hydrav1.ScaleDownIdlePreviewDeploymentsRequest) (*hydrav1.ScaleDownIdlePreviewDeploymentsResponse, error) { +func (w *Workflow) ScaleDownIdlePreviewDeployments(ctx restate.ObjectContext, req *hydrav1.ScaleDownIdlePreviewDeploymentsRequest) (*hydrav1.ScaleDownIdlePreviewDeploymentsResponse, error) { cutoff := time.Now().Add(-idleTime).UnixMilli() diff --git a/svc/ctrl/worker/githubwebhook/handle_push.go b/svc/ctrl/worker/githubwebhook/handle_push.go index bd18ac25235..89bfc616447 100644 --- a/svc/ctrl/worker/githubwebhook/handle_push.go +++ b/svc/ctrl/worker/githubwebhook/handle_push.go @@ -167,8 +167,8 @@ func (s *Service) HandlePush(ctx restate.ObjectContext, req *hydrav1.HandlePushR "environment", env.Slug, ) - // Start deploy workflow keyed by project ID - deployClient := hydrav1.NewDeployServiceClient(ctx, project.ID) + // Start deploy workflow keyed by workspace ID, to run 1 concurrent build per workspace for now during beta + deployClient := hydrav1.NewDeployServiceClient(ctx, app.WorkspaceID) deployClient.Deploy().Send(&hydrav1.DeployRequest{ DeploymentId: deploymentID, Source: &hydrav1.DeployRequest_Git{ diff --git a/svc/ctrl/worker/routing/assign_domains_handler.go b/svc/ctrl/worker/routing/assign_domains_handler.go index b2763b95c51..c17ca432fd2 100644 --- a/svc/ctrl/worker/routing/assign_domains_handler.go +++ b/svc/ctrl/worker/routing/assign_domains_handler.go @@ -23,7 +23,7 @@ import ( func (s *Service) AssignFrontlineRoutes(ctx restate.ObjectContext, req *hydrav1.AssignFrontlineRoutesRequest) (*hydrav1.AssignFrontlineRoutesResponse, error) { logger.Info("assigning domains", "deployment_id", req.GetDeploymentId(), - "frontline_route_count", len(req.GetFrontlineRouteIds()), + "frontline_routes", req.GetFrontlineRouteIds(), ) // Upsert each domain in the database diff --git a/svc/frontline/services/caches/caches.go b/svc/frontline/services/caches/caches.go index a63d1ac2a5c..06b8c6e54c4 100644 --- a/svc/frontline/services/caches/caches.go +++ b/svc/frontline/services/caches/caches.go @@ -129,7 +129,7 @@ func New(config Config) (*Caches, error) { frontlineRoute, err := createCache( cache.Config[string, db.FrontlineRoute]{ - Fresh: 30 * time.Second, + Fresh: 5 * time.Second, Stale: 5 * time.Minute, MaxSize: 10_000, Resource: "frontline_route", @@ -143,7 +143,7 @@ func New(config Config) (*Caches, error) { sentinelsByEnvironment, err := createCache( cache.Config[string, []db.Sentinel]{ - Fresh: 30 * time.Second, + Fresh: 5 * time.Second, Stale: 2 * time.Minute, MaxSize: 10_000, Resource: "sentinels_by_environment", diff --git a/web/internal/db/src/schema/frontline_routes.ts b/web/internal/db/src/schema/frontline_routes.ts index 34d5196a584..e03abe92e0d 100644 --- a/web/internal/db/src/schema/frontline_routes.ts +++ b/web/internal/db/src/schema/frontline_routes.ts @@ -20,10 +20,10 @@ export const frontlineRoutes = mysqlTable( .unique(), // sticky determines whether a fullyQualifiedDomainName should get reassigned to the latest deployment // - branch: the fullyQualifiedDomainName always points to the latest deployment on the branch - // -git--.unkey.app + // --git--.unkey.app // // - environment: the fullyQualifiedDomainName is sticky to the environment it was created on - // --.unkey.app + // ---.unkey.app // // - live: the fullyQualifiedDomainName is sticky to the live deployment it was created on // api.unkey.com From 195f5d6139ec24b96696d43525b5da554f5d21af Mon Sep 17 00:00:00 2001 From: chronark Date: Sat, 7 Mar 2026 11:53:17 +0100 Subject: [PATCH 02/24] feat: scaling and region tables --- pkg/db/models_generated.go | 31 +++++++++ pkg/db/schema.sql | 40 +++++++++++ .../db/src/schema/app_runtime_settings.ts | 3 + .../db/src/schema/app_scaling_settings.ts | 67 +++++++++++++++++++ web/internal/db/src/schema/cluster_regions.ts | 20 ++++++ web/internal/db/src/schema/clusters.ts | 26 +++++++ web/internal/db/src/schema/index.ts | 5 ++ 7 files changed, 192 insertions(+) create mode 100644 web/internal/db/src/schema/app_scaling_settings.ts create mode 100644 web/internal/db/src/schema/cluster_regions.ts create mode 100644 web/internal/db/src/schema/clusters.ts diff --git a/pkg/db/models_generated.go b/pkg/db/models_generated.go index 157aaf0a56f..15185513130 100644 --- a/pkg/db/models_generated.go +++ b/pkg/db/models_generated.go @@ -1019,6 +1019,23 @@ type AppRuntimeSetting struct { UpdatedAt sql.NullInt64 `db:"updated_at"` } +type AppScalingSetting struct { + Pk uint64 `db:"pk"` + WorkspaceID string `db:"workspace_id"` + AppID string `db:"app_id"` + EnvironmentID string `db:"environment_id"` + RegionID string `db:"region_id"` + MemoryThreshold sql.NullInt16 `db:"memory_threshold"` + CpuThreshold sql.NullInt16 `db:"cpu_threshold"` + RpsThreshold sql.NullInt16 `db:"rps_threshold"` + ReplicasMin int32 `db:"replicas_min"` + ReplicasMax int32 `db:"replicas_max"` + MaxCpuMillicores int32 `db:"max_cpu_millicores"` + MaxMemoryMib int32 `db:"max_memory_mib"` + CreatedAt int64 `db:"created_at"` + UpdatedAt sql.NullInt64 `db:"updated_at"` +} + type AuditLog struct { Pk uint64 `db:"pk"` ID string `db:"id"` @@ -1107,6 +1124,20 @@ type ClickhouseWorkspaceSetting struct { UpdatedAt sql.NullInt64 `db:"updated_at"` } +type Cluster struct { + Pk uint64 `db:"pk"` + ID string `db:"id"` + RegionID string `db:"region_id"` + LastHeartbeatAt uint64 `db:"last_heartbeat_at"` +} + +type ClusterRegion struct { + Pk uint64 `db:"pk"` + ID string `db:"id"` + Name string `db:"name"` + Platform string `db:"platform"` +} + type CustomDomain struct { Pk uint64 `db:"pk"` ID string `db:"id"` diff --git a/pkg/db/schema.sql b/pkg/db/schema.sql index 63c25e1c5cc..916d6a6406c 100644 --- a/pkg/db/schema.sql +++ b/pkg/db/schema.sql @@ -448,6 +448,25 @@ CREATE TABLE `app_runtime_settings` ( CONSTRAINT `app_runtime_settings_app_env_idx` UNIQUE(`app_id`,`environment_id`) ); +CREATE TABLE `app_scaling_settings` ( + `pk` bigint unsigned AUTO_INCREMENT NOT NULL, + `workspace_id` varchar(256) NOT NULL, + `app_id` varchar(64) NOT NULL, + `environment_id` varchar(128) NOT NULL, + `region_id` varchar(64) NOT NULL, + `memory_threshold` tinyint, + `cpu_threshold` tinyint, + `rps_threshold` tinyint, + `replicas_min` int NOT NULL, + `replicas_max` int NOT NULL, + `max_cpu_millicores` int NOT NULL, + `max_memory_mib` int NOT NULL, + `created_at` bigint NOT NULL, + `updated_at` bigint, + CONSTRAINT `app_scaling_settings_pk` PRIMARY KEY(`pk`), + CONSTRAINT `unique_app_env_region` UNIQUE(`app_id`,`environment_id`,`region_id`) +); + CREATE TABLE `app_environment_variables` ( `pk` bigint unsigned AUTO_INCREMENT NOT NULL, `id` varchar(128) NOT NULL, @@ -704,6 +723,26 @@ CREATE TABLE `cilium_network_policies` ( CONSTRAINT `unique_version_per_region` UNIQUE(`region`,`version`) ); +CREATE TABLE `clusters` ( + `pk` bigint unsigned AUTO_INCREMENT NOT NULL, + `id` varchar(64) NOT NULL, + `region_id` varchar(64) NOT NULL, + `last_heartbeat_at` bigint unsigned NOT NULL, + CONSTRAINT `clusters_pk` PRIMARY KEY(`pk`), + CONSTRAINT `clusters_id_unique` UNIQUE(`id`) +); + +CREATE TABLE `cluster_regions` ( + `pk` bigint unsigned AUTO_INCREMENT NOT NULL, + `id` varchar(64) NOT NULL, + `name` varchar(64) NOT NULL, + `platform` varchar(64) NOT NULL, + CONSTRAINT `cluster_regions_pk` PRIMARY KEY(`pk`), + CONSTRAINT `cluster_regions_id_unique` UNIQUE(`id`), + CONSTRAINT `cluster_regions_name_unique` UNIQUE(`name`), + CONSTRAINT `unique_reqion_per_platform` UNIQUE(`name`,`platform`) +); + CREATE INDEX `workspace_id_idx` ON `apis` (`workspace_id`); CREATE INDEX `workspace_id_idx` ON `roles` (`workspace_id`); CREATE INDEX `key_auth_id_deleted_at_idx` ON `keys` (`key_auth_id`,`deleted_at_m`); @@ -723,6 +762,7 @@ CREATE INDEX `bucket` ON `audit_log_target` (`bucket`); CREATE INDEX `id_idx` ON `audit_log_target` (`id`); CREATE INDEX `environments_project_idx` ON `environments` (`project_id`); CREATE INDEX `apps_workspace_idx` ON `apps` (`workspace_id`); +CREATE INDEX `workspace_idx` ON `app_scaling_settings` (`workspace_id`); CREATE INDEX `workspace_idx` ON `deployments` (`workspace_id`); CREATE INDEX `project_idx` ON `deployments` (`project_id`); CREATE INDEX `status_idx` ON `deployments` (`status`); diff --git a/web/internal/db/src/schema/app_runtime_settings.ts b/web/internal/db/src/schema/app_runtime_settings.ts index 3492e902e30..669044f9279 100644 --- a/web/internal/db/src/schema/app_runtime_settings.ts +++ b/web/internal/db/src/schema/app_runtime_settings.ts @@ -33,7 +33,9 @@ export const appRuntimeSettings = mysqlTable( environmentId: varchar("environment_id", { length: 128 }).notNull(), port: int("port").notNull().default(8080), + // @deprecated, use appScalingSettings instead cpuMillicores: int("cpu_millicores").notNull().default(256), + // @deprecated, use appScalingSettings instead memoryMib: int("memory_mib").notNull().default(256), command: json("command").$type().notNull().default(sql`('[]')`), @@ -42,6 +44,7 @@ export const appRuntimeSettings = mysqlTable( // Maps region ID to replica count, e.g. {"us-east-1": 3, "eu-central-1": 1} // Empty object = 1 replica in all available regions (default behavior) + // @deprecated, use appScalingSettings instead regionConfig: json("region_config") .$type>() .notNull() diff --git a/web/internal/db/src/schema/app_scaling_settings.ts b/web/internal/db/src/schema/app_scaling_settings.ts new file mode 100644 index 00000000000..cd9b67902b0 --- /dev/null +++ b/web/internal/db/src/schema/app_scaling_settings.ts @@ -0,0 +1,67 @@ +import { relations } from "drizzle-orm"; +import { + bigint, + index, + int, + mysqlTable, + tinyint, + uniqueIndex, + varchar, +} from "drizzle-orm/mysql-core"; +import { apps } from "./apps"; +import { environments } from "./environments"; +import { lifecycleDates } from "./util/lifecycle_dates"; +import { workspaces } from "./workspaces"; + +/** + * One row per region an app+environment should be deployed to. + * Replaces the `region_config` JSON column on `app_runtime_settings`. + * + * Autoscaling is considered enabled if replicasMax > replicasMin and there is at least one threshold configured (memoryThreshold, cpuThreshold, or rpsThreshold). + * If autoscaling is enabled, the system will scale up the app when any of the specified thresholds are breached, and scale down when they are no longer breached, + * within the limits of replicasMin and replicasMax. + */ +export const appScalingSettings = mysqlTable( + "app_scaling_settings", + { + pk: bigint("pk", { mode: "number", unsigned: true }).autoincrement().primaryKey(), + + workspaceId: varchar("workspace_id", { length: 256 }).notNull(), + appId: varchar("app_id", { length: 64 }).notNull(), + environmentId: varchar("environment_id", { length: 128 }).notNull(), + regionId: varchar("region_id", { length: 64 }).notNull(), + + // 0-100, the percentage threshold that triggers scaling + memoryThreshold: tinyint("memory_threshold"), + cpuThreshold: tinyint("cpu_threshold"), + rpsThreshold: tinyint("rps_threshold"), + + // set both replicasMin and replicasMax to the same value to disable autoscaling + replicasMin: int("replicas_min").notNull(), + replicasMax: int("replicas_max").notNull(), + + cpuMillicores: int("max_cpu_millicores").notNull(), + memoryMib: int("max_memory_mib").notNull(), + + ...lifecycleDates, + }, + (table) => [ + uniqueIndex("unique_app_env_region").on(table.appId, table.environmentId, table.regionId), + index("workspace_idx").on(table.workspaceId), + ], +); + +export const appScalingSettingsRelations = relations(appScalingSettings, ({ one }) => ({ + workspace: one(workspaces, { + fields: [appScalingSettings.workspaceId], + references: [workspaces.id], + }), + app: one(apps, { + fields: [appScalingSettings.appId], + references: [apps.id], + }), + environment: one(environments, { + fields: [appScalingSettings.environmentId], + references: [environments.id], + }), +})); diff --git a/web/internal/db/src/schema/cluster_regions.ts b/web/internal/db/src/schema/cluster_regions.ts new file mode 100644 index 00000000000..d9423349770 --- /dev/null +++ b/web/internal/db/src/schema/cluster_regions.ts @@ -0,0 +1,20 @@ +import { relations } from "drizzle-orm"; +import { bigint, mysqlTable, uniqueIndex, varchar } from "drizzle-orm/mysql-core"; +import { clusters } from "./clusters"; + +export const clusterRegions = mysqlTable( + "cluster_regions", + { + pk: bigint("pk", { mode: "number", unsigned: true }).autoincrement().primaryKey(), + id: varchar("id", { length: 64 }).notNull().unique(), + // e.g. us-east-1, us-west-2, etc. + name: varchar("name", { length: 64 }).unique().notNull(), + // e.g. aws, gcp, azure, local, etc. + platform: varchar("platform", { length: 64 }).notNull(), + }, + (table) => [uniqueIndex("unique_reqion_per_platform").on(table.name, table.platform)], +); + +export const clusterRegionRelations = relations(clusterRegions, ({ many }) => ({ + clusters: many(clusters), +})); diff --git a/web/internal/db/src/schema/clusters.ts b/web/internal/db/src/schema/clusters.ts new file mode 100644 index 00000000000..ee68f8c0ab8 --- /dev/null +++ b/web/internal/db/src/schema/clusters.ts @@ -0,0 +1,26 @@ +import { relations } from "drizzle-orm"; +import { bigint, mysqlTable, varchar } from "drizzle-orm/mysql-core"; +import { clusterRegions } from "./cluster_regions"; + +// clusters tracks our kubernetes clusters +// each krane instance will heartbeat against the control plane, which then writes to this table +// +// we might use this as service discovery later to push updates to clusters to speed up reconciliation +export const clusters = mysqlTable("clusters", { + pk: bigint("pk", { mode: "number", unsigned: true }).autoincrement().primaryKey(), + + id: varchar("id", { length: 64 }).notNull().unique(), + regionId: varchar("region_id", { length: 64 }).notNull(), + + lastHeartbeatAt: bigint("last_heartbeat_at", { + mode: "number", + unsigned: true, + }).notNull(), +}); + +export const regionsRelations = relations(clusters, ({ one }) => ({ + workspace: one(clusterRegions, { + fields: [clusters.regionId], + references: [clusterRegions.id], + }), +})); diff --git a/web/internal/db/src/schema/index.ts b/web/internal/db/src/schema/index.ts index 415bd9b0579..e0deae2aba3 100644 --- a/web/internal/db/src/schema/index.ts +++ b/web/internal/db/src/schema/index.ts @@ -17,6 +17,8 @@ export * from "./projects"; export * from "./apps"; export * from "./app_build_settings"; export * from "./app_runtime_settings"; +export * from "./app_scaling_settings"; + export * from "./app_environment_variables"; export * from "./deployments"; export * from "./deployment_steps"; @@ -32,3 +34,6 @@ export * from "./certificates"; export * from "./frontline_routes"; export * from "./github_app"; export * from "./cilium"; + +export * from "./clusters"; +export * from "./cluster_regions"; From ccebb8c97b5fd3d9aac135d275ad3fb6b635c397 Mon Sep 17 00:00:00 2001 From: chronark Date: Sat, 7 Mar 2026 12:00:25 +0100 Subject: [PATCH 03/24] fix: replace AVAILABLE_REGIONS env var with clusterRegions table query --- .../environment-settings/get-available-regions.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/get-available-regions.ts b/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/get-available-regions.ts index bc0999979b7..16337bf3be6 100644 --- a/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/get-available-regions.ts +++ b/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/get-available-regions.ts @@ -1,9 +1,9 @@ +import { db } from "@/lib/db"; import { workspaceProcedure } from "../../../trpc"; -export const getAvailableRegions = workspaceProcedure.query(() => { - const regionsEnv = process.env.AVAILABLE_REGIONS ?? ""; - return regionsEnv - .split(",") - .map((r) => r.trim()) - .filter(Boolean); +export const getAvailableRegions = workspaceProcedure.query(async () => { + const regions = await db.query.clusterRegions.findMany({ + columns: { name: true }, + }); + return regions.map((r) => r.name); }); From 2bcd687f70a7842903499b30b6054aa91cdac808 Mon Sep 17 00:00:00 2001 From: chronark Date: Sat, 7 Mar 2026 12:03:05 +0100 Subject: [PATCH 04/24] fix: update instances mutation to use appScalingSettings table --- .../runtime/update-instances.ts | 36 +++++-------------- 1 file changed, 8 insertions(+), 28 deletions(-) diff --git a/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/runtime/update-instances.ts b/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/runtime/update-instances.ts index 369cf7842c3..4b8ddadc855 100644 --- a/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/runtime/update-instances.ts +++ b/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/runtime/update-instances.ts @@ -1,5 +1,5 @@ import { and, db, eq, inArray } from "@/lib/db"; -import { appRuntimeSettings } from "@unkey/db/src/schema"; +import { appScalingSettings } from "@unkey/db/src/schema"; import { z } from "zod"; import { workspaceProcedure } from "../../../../trpc"; import { resolveProjectEnvironmentIds } from "../utils"; @@ -14,36 +14,16 @@ export const updateInstances = workspaceProcedure .mutation(async ({ ctx, input }) => { const envIds = await resolveProjectEnvironmentIds(ctx.workspace.id, input.environmentId); - const existing = await db.query.appRuntimeSettings.findFirst({ - where: and( - eq(appRuntimeSettings.workspaceId, ctx.workspace.id), - eq(appRuntimeSettings.environmentId, input.environmentId), - ), - }); - - const currentConfig = (existing?.regionConfig as Record) ?? {}; - const currentRegions = Object.keys(currentConfig); - - const regionConfig: Record = {}; - - if (currentRegions.length > 0) { - for (const region of currentRegions) { - regionConfig[region] = input.replicasPerRegion; - } - } else { - const regionsEnv = process.env.AVAILABLE_REGIONS ?? ""; - for (const region of regionsEnv.split(",")) { - regionConfig[region] = input.replicasPerRegion; - } - } - await db - .update(appRuntimeSettings) - .set({ regionConfig }) + .update(appScalingSettings) + .set({ + replicasMin: input.replicasPerRegion, + replicasMax: input.replicasPerRegion, + }) .where( and( - eq(appRuntimeSettings.workspaceId, ctx.workspace.id), - inArray(appRuntimeSettings.environmentId, envIds), + eq(appScalingSettings.workspaceId, ctx.workspace.id), + inArray(appScalingSettings.environmentId, envIds), ), ); }); From aa92057c1437d13c020cf0025e61c877713f0351 Mon Sep 17 00:00:00 2001 From: chronark Date: Sat, 7 Mar 2026 12:04:15 +0100 Subject: [PATCH 05/24] fix: update CPU mutation to use appScalingSettings table --- .../deploy/environment-settings/runtime/update-cpu.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/runtime/update-cpu.ts b/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/runtime/update-cpu.ts index d9e7e4dbb5b..04d5bc57101 100644 --- a/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/runtime/update-cpu.ts +++ b/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/runtime/update-cpu.ts @@ -1,5 +1,5 @@ import { and, db, eq, inArray } from "@/lib/db"; -import { appRuntimeSettings } from "@unkey/db/src/schema"; +import { appScalingSettings } from "@unkey/db/src/schema"; import { z } from "zod"; import { workspaceProcedure } from "../../../../trpc"; import { resolveProjectEnvironmentIds } from "../utils"; @@ -15,12 +15,12 @@ export const updateCpu = workspaceProcedure const envIds = await resolveProjectEnvironmentIds(ctx.workspace.id, input.environmentId); await db - .update(appRuntimeSettings) + .update(appScalingSettings) .set({ cpuMillicores: input.cpuMillicores }) .where( and( - eq(appRuntimeSettings.workspaceId, ctx.workspace.id), - inArray(appRuntimeSettings.environmentId, envIds), + eq(appScalingSettings.workspaceId, ctx.workspace.id), + inArray(appScalingSettings.environmentId, envIds), ), ); }); From 47a4ae12f6c8c3263e874b05ac4f463eda2c8c29 Mon Sep 17 00:00:00 2001 From: chronark Date: Sat, 7 Mar 2026 12:17:17 +0100 Subject: [PATCH 06/24] fix: update memory mutation to use appScalingSettings table --- .../deploy/environment-settings/runtime/update-memory.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/runtime/update-memory.ts b/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/runtime/update-memory.ts index ba77616fa47..ddf70df352f 100644 --- a/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/runtime/update-memory.ts +++ b/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/runtime/update-memory.ts @@ -1,5 +1,5 @@ import { and, db, eq, inArray } from "@/lib/db"; -import { appRuntimeSettings } from "@unkey/db/src/schema"; +import { appScalingSettings } from "@unkey/db/src/schema"; import { z } from "zod"; import { workspaceProcedure } from "../../../../trpc"; import { resolveProjectEnvironmentIds } from "../utils"; @@ -15,12 +15,12 @@ export const updateMemory = workspaceProcedure const envIds = await resolveProjectEnvironmentIds(ctx.workspace.id, input.environmentId); await db - .update(appRuntimeSettings) + .update(appScalingSettings) .set({ memoryMib: input.memoryMib }) .where( and( - eq(appRuntimeSettings.workspaceId, ctx.workspace.id), - inArray(appRuntimeSettings.environmentId, envIds), + eq(appScalingSettings.workspaceId, ctx.workspace.id), + inArray(appScalingSettings.environmentId, envIds), ), ); }); From 325e89c283777f7fa4792661fcec7dea4fdb81ae Mon Sep 17 00:00:00 2001 From: chronark Date: Sat, 7 Mar 2026 12:21:18 +0100 Subject: [PATCH 07/24] fix: update regions mutation to use appScalingSettings and clusterRegions tables --- .../runtime/update-regions.ts | 81 +++++++++++++++---- 1 file changed, 65 insertions(+), 16 deletions(-) diff --git a/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/runtime/update-regions.ts b/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/runtime/update-regions.ts index ef4a557a039..f249c817696 100644 --- a/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/runtime/update-regions.ts +++ b/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/runtime/update-regions.ts @@ -1,6 +1,7 @@ -import { and, db, eq, inArray } from "@/lib/db"; -import { appRuntimeSettings } from "@unkey/db/src/schema"; +import { and, db, eq, inArray, notInArray } from "@/lib/db"; +import { appScalingSettings, clusterRegions, environments } from "@unkey/db/src/schema"; import { z } from "zod"; +import { TRPCError } from "@trpc/server"; import { workspaceProcedure } from "../../../../trpc"; import { resolveProjectEnvironmentIds } from "../utils"; @@ -14,26 +15,74 @@ export const updateRegions = workspaceProcedure .mutation(async ({ ctx, input }) => { const envIds = await resolveProjectEnvironmentIds(ctx.workspace.id, input.environmentId); - const existing = await db.query.appRuntimeSettings.findFirst({ + // Resolve region names to IDs + const regions = await db.query.clusterRegions.findMany({ + where: inArray(clusterRegions.name, input.regions), + columns: { id: true, name: true }, + }); + const regionIds = regions.map((r) => r.id); + + if (regionIds.length !== input.regions.length) { + const found = new Set(regions.map((r) => r.name)); + const missing = input.regions.filter((r) => !found.has(r)); + throw new TRPCError({ + code: "BAD_REQUEST", + message: `Unknown regions: ${missing.join(", ")}`, + }); + } + + // Get appId from the environment + const env = await db.query.environments.findFirst({ where: and( - eq(appRuntimeSettings.workspaceId, ctx.workspace.id), - eq(appRuntimeSettings.environmentId, input.environmentId), + eq(environments.id, input.environmentId), + eq(environments.workspaceId, ctx.workspace.id), ), + columns: { appId: true }, }); - - const currentConfig = (existing?.regionConfig as Record) ?? {}; - const regionConfig: Record = {}; - for (const region of input.regions) { - regionConfig[region] = currentConfig[region] ?? 1; + if (!env) { + throw new TRPCError({ code: "NOT_FOUND", message: "Environment not found" }); } - await db - .update(appRuntimeSettings) - .set({ regionConfig }) - .where( + // Get existing scaling settings to preserve values for new regions + const existingSettings = await db.query.appScalingSettings.findMany({ + where: and( + eq(appScalingSettings.workspaceId, ctx.workspace.id), + eq(appScalingSettings.environmentId, input.environmentId), + ), + }); + + const defaults = existingSettings.at(0) ?? { + replicasMin: 1, + replicasMax: 1, + cpuMillicores: 256, + memoryMib: 256, + }; + + for (const envId of envIds) { + // Delete rows for regions that are no longer selected + await db.delete(appScalingSettings).where( and( - eq(appRuntimeSettings.workspaceId, ctx.workspace.id), - inArray(appRuntimeSettings.environmentId, envIds), + eq(appScalingSettings.workspaceId, ctx.workspace.id), + eq(appScalingSettings.environmentId, envId), + notInArray(appScalingSettings.regionId, regionIds), ), ); + + // Insert rows for newly added regions + for (const regionId of regionIds) { + const existing = existingSettings.find((s) => s.regionId === regionId); + if (!existing) { + await db.insert(appScalingSettings).values({ + workspaceId: ctx.workspace.id, + appId: env.appId, + environmentId: envId, + regionId, + replicasMin: defaults.replicasMin, + replicasMax: defaults.replicasMax, + cpuMillicores: defaults.cpuMillicores, + memoryMib: defaults.memoryMib, + }); + } + } + } }); From f16d7c8e0c42fcfd2dfc97c72e189fa86ed2bfad Mon Sep 17 00:00:00 2001 From: chronark Date: Sat, 7 Mar 2026 12:36:49 +0100 Subject: [PATCH 08/24] fix: revert cpu/memory mutations back to appRuntimeSettings --- .../deploy/environment-settings/runtime/update-cpu.ts | 8 ++++---- .../deploy/environment-settings/runtime/update-memory.ts | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/runtime/update-cpu.ts b/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/runtime/update-cpu.ts index 04d5bc57101..d9e7e4dbb5b 100644 --- a/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/runtime/update-cpu.ts +++ b/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/runtime/update-cpu.ts @@ -1,5 +1,5 @@ import { and, db, eq, inArray } from "@/lib/db"; -import { appScalingSettings } from "@unkey/db/src/schema"; +import { appRuntimeSettings } from "@unkey/db/src/schema"; import { z } from "zod"; import { workspaceProcedure } from "../../../../trpc"; import { resolveProjectEnvironmentIds } from "../utils"; @@ -15,12 +15,12 @@ export const updateCpu = workspaceProcedure const envIds = await resolveProjectEnvironmentIds(ctx.workspace.id, input.environmentId); await db - .update(appScalingSettings) + .update(appRuntimeSettings) .set({ cpuMillicores: input.cpuMillicores }) .where( and( - eq(appScalingSettings.workspaceId, ctx.workspace.id), - inArray(appScalingSettings.environmentId, envIds), + eq(appRuntimeSettings.workspaceId, ctx.workspace.id), + inArray(appRuntimeSettings.environmentId, envIds), ), ); }); diff --git a/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/runtime/update-memory.ts b/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/runtime/update-memory.ts index ddf70df352f..ba77616fa47 100644 --- a/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/runtime/update-memory.ts +++ b/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/runtime/update-memory.ts @@ -1,5 +1,5 @@ import { and, db, eq, inArray } from "@/lib/db"; -import { appScalingSettings } from "@unkey/db/src/schema"; +import { appRuntimeSettings } from "@unkey/db/src/schema"; import { z } from "zod"; import { workspaceProcedure } from "../../../../trpc"; import { resolveProjectEnvironmentIds } from "../utils"; @@ -15,12 +15,12 @@ export const updateMemory = workspaceProcedure const envIds = await resolveProjectEnvironmentIds(ctx.workspace.id, input.environmentId); await db - .update(appScalingSettings) + .update(appRuntimeSettings) .set({ memoryMib: input.memoryMib }) .where( and( - eq(appScalingSettings.workspaceId, ctx.workspace.id), - inArray(appScalingSettings.environmentId, envIds), + eq(appRuntimeSettings.workspaceId, ctx.workspace.id), + inArray(appRuntimeSettings.environmentId, envIds), ), ); }); From 0112d647cff1b897d4beedace63f41f0cdfe9b03 Mon Sep 17 00:00:00 2001 From: chronark Date: Sat, 7 Mar 2026 12:44:57 +0100 Subject: [PATCH 09/24] feat: rename app_scaling_settings to app_regional_settings, add horizontal_autoscaling_policies table --- .../runtime/update-instances.ts | 11 ++- .../runtime/update-regions.ts | 28 +++----- .../db/src/schema/app_regional_settings.ts | 66 ++++++++++++++++++ .../db/src/schema/app_runtime_settings.ts | 4 +- .../db/src/schema/app_scaling_settings.ts | 67 ------------------- .../schema/horizontal_autoscaling_policies.ts | 47 +++++++++++++ web/internal/db/src/schema/index.ts | 3 +- 7 files changed, 132 insertions(+), 94 deletions(-) create mode 100644 web/internal/db/src/schema/app_regional_settings.ts delete mode 100644 web/internal/db/src/schema/app_scaling_settings.ts create mode 100644 web/internal/db/src/schema/horizontal_autoscaling_policies.ts diff --git a/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/runtime/update-instances.ts b/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/runtime/update-instances.ts index 4b8ddadc855..b3f5919a0fd 100644 --- a/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/runtime/update-instances.ts +++ b/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/runtime/update-instances.ts @@ -1,5 +1,5 @@ import { and, db, eq, inArray } from "@/lib/db"; -import { appScalingSettings } from "@unkey/db/src/schema"; +import { appRegionalSettings } from "@unkey/db/src/schema"; import { z } from "zod"; import { workspaceProcedure } from "../../../../trpc"; import { resolveProjectEnvironmentIds } from "../utils"; @@ -15,15 +15,14 @@ export const updateInstances = workspaceProcedure const envIds = await resolveProjectEnvironmentIds(ctx.workspace.id, input.environmentId); await db - .update(appScalingSettings) + .update(appRegionalSettings) .set({ - replicasMin: input.replicasPerRegion, - replicasMax: input.replicasPerRegion, + replicas: input.replicasPerRegion, }) .where( and( - eq(appScalingSettings.workspaceId, ctx.workspace.id), - inArray(appScalingSettings.environmentId, envIds), + eq(appRegionalSettings.workspaceId, ctx.workspace.id), + inArray(appRegionalSettings.environmentId, envIds), ), ); }); diff --git a/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/runtime/update-regions.ts b/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/runtime/update-regions.ts index f249c817696..c24748b32c6 100644 --- a/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/runtime/update-regions.ts +++ b/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/runtime/update-regions.ts @@ -1,5 +1,5 @@ import { and, db, eq, inArray, notInArray } from "@/lib/db"; -import { appScalingSettings, clusterRegions, environments } from "@unkey/db/src/schema"; +import { appRegionalSettings, clusterRegions, environments } from "@unkey/db/src/schema"; import { z } from "zod"; import { TRPCError } from "@trpc/server"; import { workspaceProcedure } from "../../../../trpc"; @@ -44,27 +44,24 @@ export const updateRegions = workspaceProcedure } // Get existing scaling settings to preserve values for new regions - const existingSettings = await db.query.appScalingSettings.findMany({ + const existingSettings = await db.query.appRegionalSettings.findMany({ where: and( - eq(appScalingSettings.workspaceId, ctx.workspace.id), - eq(appScalingSettings.environmentId, input.environmentId), + eq(appRegionalSettings.workspaceId, ctx.workspace.id), + eq(appRegionalSettings.environmentId, input.environmentId), ), }); const defaults = existingSettings.at(0) ?? { - replicasMin: 1, - replicasMax: 1, - cpuMillicores: 256, - memoryMib: 256, + replicas: 1, }; for (const envId of envIds) { // Delete rows for regions that are no longer selected - await db.delete(appScalingSettings).where( + await db.delete(appRegionalSettings).where( and( - eq(appScalingSettings.workspaceId, ctx.workspace.id), - eq(appScalingSettings.environmentId, envId), - notInArray(appScalingSettings.regionId, regionIds), + eq(appRegionalSettings.workspaceId, ctx.workspace.id), + eq(appRegionalSettings.environmentId, envId), + notInArray(appRegionalSettings.regionId, regionIds), ), ); @@ -72,15 +69,12 @@ export const updateRegions = workspaceProcedure for (const regionId of regionIds) { const existing = existingSettings.find((s) => s.regionId === regionId); if (!existing) { - await db.insert(appScalingSettings).values({ + await db.insert(appRegionalSettings).values({ workspaceId: ctx.workspace.id, appId: env.appId, environmentId: envId, regionId, - replicasMin: defaults.replicasMin, - replicasMax: defaults.replicasMax, - cpuMillicores: defaults.cpuMillicores, - memoryMib: defaults.memoryMib, + replicas: defaults.replicas, }); } } diff --git a/web/internal/db/src/schema/app_regional_settings.ts b/web/internal/db/src/schema/app_regional_settings.ts new file mode 100644 index 00000000000..d93c73922e4 --- /dev/null +++ b/web/internal/db/src/schema/app_regional_settings.ts @@ -0,0 +1,66 @@ +import { relations } from "drizzle-orm"; +import { + bigint, + index, + int, + mysqlTable, + uniqueIndex, + varchar, +} from "drizzle-orm/mysql-core"; +import { apps } from "./apps"; +import { environments } from "./environments"; +import { lifecycleDates } from "./util/lifecycle_dates"; +import { workspaces } from "./workspaces"; + +/** + * One row per region an app+environment should be deployed to. + * Replaces the `region_config` JSON column on `app_runtime_settings`. + * + * Presence of a row means "deploy this app/env to this region". + */ +export const appRegionalSettings = mysqlTable( + "app_regional_settings", + { + pk: bigint("pk", { mode: "number", unsigned: true }) + .autoincrement() + .primaryKey(), + + workspaceId: varchar("workspace_id", { length: 256 }).notNull(), + appId: varchar("app_id", { length: 64 }).notNull(), + environmentId: varchar("environment_id", { length: 128 }).notNull(), + regionId: varchar("region_id", { length: 64 }).notNull(), + + replicas: int("replicas").notNull().default(1), + + // Optional reference to a horizontal autoscaling policy. null = no autoscaling. + horizontalAutoscalingPolicyId: varchar("horizontal_autoscaling_policy_id", { length: 64 }), + + ...lifecycleDates, + }, + (table) => [ + uniqueIndex("unique_app_env_region").on( + table.appId, + table.environmentId, + table.regionId, + ), + index("workspace_idx").on(table.workspaceId), + ], +); + +export const appRegionalSettingsRelations = relations( + appRegionalSettings, + ({ one }) => ({ + workspace: one(workspaces, { + fields: [appRegionalSettings.workspaceId], + references: [workspaces.id], + }), + app: one(apps, { + fields: [appRegionalSettings.appId], + references: [apps.id], + }), + environment: one(environments, { + fields: [appRegionalSettings.environmentId], + references: [environments.id], + }), + }), +); diff --git a/web/internal/db/src/schema/app_runtime_settings.ts b/web/internal/db/src/schema/app_runtime_settings.ts index 669044f9279..aa7b0ddb4d7 100644 --- a/web/internal/db/src/schema/app_runtime_settings.ts +++ b/web/internal/db/src/schema/app_runtime_settings.ts @@ -33,9 +33,7 @@ export const appRuntimeSettings = mysqlTable( environmentId: varchar("environment_id", { length: 128 }).notNull(), port: int("port").notNull().default(8080), - // @deprecated, use appScalingSettings instead cpuMillicores: int("cpu_millicores").notNull().default(256), - // @deprecated, use appScalingSettings instead memoryMib: int("memory_mib").notNull().default(256), command: json("command").$type().notNull().default(sql`('[]')`), @@ -44,7 +42,7 @@ export const appRuntimeSettings = mysqlTable( // Maps region ID to replica count, e.g. {"us-east-1": 3, "eu-central-1": 1} // Empty object = 1 replica in all available regions (default behavior) - // @deprecated, use appScalingSettings instead + // @deprecated, use appRegionalSettings instead regionConfig: json("region_config") .$type>() .notNull() diff --git a/web/internal/db/src/schema/app_scaling_settings.ts b/web/internal/db/src/schema/app_scaling_settings.ts deleted file mode 100644 index cd9b67902b0..00000000000 --- a/web/internal/db/src/schema/app_scaling_settings.ts +++ /dev/null @@ -1,67 +0,0 @@ -import { relations } from "drizzle-orm"; -import { - bigint, - index, - int, - mysqlTable, - tinyint, - uniqueIndex, - varchar, -} from "drizzle-orm/mysql-core"; -import { apps } from "./apps"; -import { environments } from "./environments"; -import { lifecycleDates } from "./util/lifecycle_dates"; -import { workspaces } from "./workspaces"; - -/** - * One row per region an app+environment should be deployed to. - * Replaces the `region_config` JSON column on `app_runtime_settings`. - * - * Autoscaling is considered enabled if replicasMax > replicasMin and there is at least one threshold configured (memoryThreshold, cpuThreshold, or rpsThreshold). - * If autoscaling is enabled, the system will scale up the app when any of the specified thresholds are breached, and scale down when they are no longer breached, - * within the limits of replicasMin and replicasMax. - */ -export const appScalingSettings = mysqlTable( - "app_scaling_settings", - { - pk: bigint("pk", { mode: "number", unsigned: true }).autoincrement().primaryKey(), - - workspaceId: varchar("workspace_id", { length: 256 }).notNull(), - appId: varchar("app_id", { length: 64 }).notNull(), - environmentId: varchar("environment_id", { length: 128 }).notNull(), - regionId: varchar("region_id", { length: 64 }).notNull(), - - // 0-100, the percentage threshold that triggers scaling - memoryThreshold: tinyint("memory_threshold"), - cpuThreshold: tinyint("cpu_threshold"), - rpsThreshold: tinyint("rps_threshold"), - - // set both replicasMin and replicasMax to the same value to disable autoscaling - replicasMin: int("replicas_min").notNull(), - replicasMax: int("replicas_max").notNull(), - - cpuMillicores: int("max_cpu_millicores").notNull(), - memoryMib: int("max_memory_mib").notNull(), - - ...lifecycleDates, - }, - (table) => [ - uniqueIndex("unique_app_env_region").on(table.appId, table.environmentId, table.regionId), - index("workspace_idx").on(table.workspaceId), - ], -); - -export const appScalingSettingsRelations = relations(appScalingSettings, ({ one }) => ({ - workspace: one(workspaces, { - fields: [appScalingSettings.workspaceId], - references: [workspaces.id], - }), - app: one(apps, { - fields: [appScalingSettings.appId], - references: [apps.id], - }), - environment: one(environments, { - fields: [appScalingSettings.environmentId], - references: [environments.id], - }), -})); diff --git a/web/internal/db/src/schema/horizontal_autoscaling_policies.ts b/web/internal/db/src/schema/horizontal_autoscaling_policies.ts new file mode 100644 index 00000000000..ee52f90a87c --- /dev/null +++ b/web/internal/db/src/schema/horizontal_autoscaling_policies.ts @@ -0,0 +1,47 @@ +import { relations } from "drizzle-orm"; +import { + bigint, + index, + int, + mysqlTable, + tinyint, + varchar, +} from "drizzle-orm/mysql-core"; +import { lifecycleDates } from "./util/lifecycle_dates"; +import { workspaces } from "./workspaces"; + +/** + * A reusable horizontal autoscaling policy. + * + * Other tables (e.g. app_regional_settings, sentinels) reference this via horizontalAutoscalingPolicyId. + * If no policy is referenced, static replica counts are used. + */ +export const horizontalAutoscalingPolicies = mysqlTable( + "horizontal_autoscaling_policies", + { + pk: bigint("pk", { mode: "number", unsigned: true }).autoincrement().primaryKey(), + id: varchar("id", { length: 64 }).notNull().unique(), + + workspaceId: varchar("workspace_id", { length: 256 }).notNull(), + + replicasMin: int("replicas_min").notNull(), + replicasMax: int("replicas_max").notNull(), + + // 0-100, percentage thresholds that trigger scaling. null = not used as a signal. + memoryThreshold: tinyint("memory_threshold"), + cpuThreshold: tinyint("cpu_threshold"), + rpsThreshold: tinyint("rps_threshold"), + + ...lifecycleDates, + }, + (table) => [ + index("workspace_idx").on(table.workspaceId), + ], +); + +export const horizontalAutoscalingPoliciesRelations = relations(horizontalAutoscalingPolicies, ({ one }) => ({ + workspace: one(workspaces, { + fields: [horizontalAutoscalingPolicies.workspaceId], + references: [workspaces.id], + }), +})); diff --git a/web/internal/db/src/schema/index.ts b/web/internal/db/src/schema/index.ts index e0deae2aba3..764274651cf 100644 --- a/web/internal/db/src/schema/index.ts +++ b/web/internal/db/src/schema/index.ts @@ -17,7 +17,7 @@ export * from "./projects"; export * from "./apps"; export * from "./app_build_settings"; export * from "./app_runtime_settings"; -export * from "./app_scaling_settings"; +export * from "./app_regional_settings"; export * from "./app_environment_variables"; export * from "./deployments"; @@ -37,3 +37,4 @@ export * from "./cilium"; export * from "./clusters"; export * from "./cluster_regions"; +export * from "./horizontal_autoscaling_policies"; From 5a6c9013444f0bb7b7321483afd16806c78904dd Mon Sep 17 00:00:00 2001 From: chronark Date: Sat, 7 Mar 2026 12:55:43 +0100 Subject: [PATCH 10/24] feat: replace regionConfig with appRegionalSettings across dashboard --- .../components/runtime-settings/instances.tsx | 14 ++-- .../components/runtime-settings/regions.tsx | 65 ++++++++++--------- .../steps/onboarding-environment-provider.tsx | 8 +-- .../deploy/environment-settings.ts | 27 ++++---- .../get-available-regions.ts | 4 +- .../deploy/environment-settings/get.ts | 11 +++- .../runtime/update-regions.ts | 26 ++------ 7 files changed, 72 insertions(+), 83 deletions(-) diff --git a/web/apps/dashboard/app/(app)/[workspaceSlug]/projects/[projectId]/(overview)/settings/components/runtime-settings/instances.tsx b/web/apps/dashboard/app/(app)/[workspaceSlug]/projects/[projectId]/(overview)/settings/components/runtime-settings/instances.tsx index 1fa1d272a30..3043d9d0079 100644 --- a/web/apps/dashboard/app/(app)/[workspaceSlug]/projects/[projectId]/(overview)/settings/components/runtime-settings/instances.tsx +++ b/web/apps/dashboard/app/(app)/[workspaceSlug]/projects/[projectId]/(overview)/settings/components/runtime-settings/instances.tsx @@ -21,10 +21,10 @@ type InstancesFormValues = z.infer; export const Instances = () => { const { settings, autoSave } = useEnvironmentSettings(); - const { environmentId, regionConfig } = settings; + const { environmentId, regions } = settings; - const selectedRegions = Object.keys(regionConfig); - const defaultInstances = Object.values(regionConfig)[0] ?? 1; + const selectedRegions = regions.map((r) => r.name); + const defaultInstances = regions.at(0)?.replicas ?? 1; const { handleSubmit, @@ -46,16 +46,14 @@ export const Instances = () => { const onSubmit = async (values: InstancesFormValues) => { collection.environmentSettings.update(environmentId, (draft) => { - const updated: Record = {}; - for (const region of Object.keys(draft.regionConfig)) { - updated[region] = values.instances; + for (const region of draft.regions) { + region.replicas = values.instances; } - draft.regionConfig = updated; }); }; const hasChanges = currentInstances !== defaultInstances; - const hasRegions = Object.keys(regionConfig).length > 0; + const hasRegions = regions.length > 0; const saveState = resolveSaveState([ [isSubmitting, { status: "saving" }], diff --git a/web/apps/dashboard/app/(app)/[workspaceSlug]/projects/[projectId]/(overview)/settings/components/runtime-settings/regions.tsx b/web/apps/dashboard/app/(app)/[workspaceSlug]/projects/[projectId]/(overview)/settings/components/runtime-settings/regions.tsx index 7bd0b3547b9..85b907addf5 100644 --- a/web/apps/dashboard/app/(app)/[workspaceSlug]/projects/[projectId]/(overview)/settings/components/runtime-settings/regions.tsx +++ b/web/apps/dashboard/app/(app)/[workspaceSlug]/projects/[projectId]/(overview)/settings/components/runtime-settings/regions.tsx @@ -20,10 +20,12 @@ const regionsSchema = z.object({ type RegionsFormValues = z.infer; +type AvailableRegion = { id: string; name: string }; + export const Regions = () => { const { settings, autoSave } = useEnvironmentSettings(); - const { environmentId, regionConfig } = settings; - const defaultRegions = Object.keys(regionConfig); + const { environmentId, regions } = settings; + const defaultRegionNames = regions.map((r) => r.name); const { data: availableRegions } = trpc.deploy.environmentSettings.getAvailableRegions.useQuery( undefined, @@ -33,7 +35,7 @@ export const Regions = () => { return ( @@ -42,14 +44,14 @@ export const Regions = () => { type RegionsFormProps = { environmentId: string; - defaultRegions: string[]; - availableRegions: string[]; + defaultRegionNames: string[]; + availableRegions: AvailableRegion[]; autoSave?: boolean; }; const RegionsForm: React.FC = ({ environmentId, - defaultRegions, + defaultRegionNames, availableRegions, autoSave, }) => { @@ -62,45 +64,48 @@ const RegionsForm: React.FC = ({ } = useForm({ resolver: zodResolver(regionsSchema), mode: "onChange", - defaultValues: { regions: defaultRegions }, + defaultValues: { regions: defaultRegionNames }, }); useEffect(() => { - reset({ regions: defaultRegions }); - }, [defaultRegions, reset]); + reset({ regions: defaultRegionNames }); + }, [defaultRegionNames, reset]); const currentRegions = useWatch({ control, name: "regions" }); - const unselectedRegions = availableRegions.filter((r) => !currentRegions.includes(r)); + const unselectedRegions = availableRegions.filter((r) => !currentRegions.includes(r.name)); const onSubmit = async (values: RegionsFormValues) => { collection.environmentSettings.update(environmentId, (draft) => { - const newConfig: Record = {}; - const defaultCount = Object.values(draft.regionConfig)[0] ?? 1; - for (const region of values.regions) { - newConfig[region] = draft.regionConfig[region] ?? defaultCount; - } - draft.regionConfig = newConfig; + const defaultReplicas = draft.regions.at(0)?.replicas ?? 1; + draft.regions = values.regions.map((name) => { + const existing = draft.regions.find((r) => r.name === name); + if (existing) { + return existing; + } + const available = availableRegions.find((r) => r.name === name); + return { id: available?.id ?? name, name, replicas: defaultReplicas }; + }); }); }; - const addRegion = (region: string) => { - if (region && !currentRegions.includes(region)) { - setValue("regions", [...currentRegions, region], { shouldValidate: true }); + const addRegion = (regionName: string) => { + if (regionName && !currentRegions.includes(regionName)) { + setValue("regions", [...currentRegions, regionName], { shouldValidate: true }); } }; - const removeRegion = (region: string) => { + const removeRegion = (regionName: string) => { setValue( "regions", - currentRegions.filter((r) => r !== region), + currentRegions.filter((r) => r !== regionName), { shouldValidate: true }, ); }; const hasChanges = - currentRegions.length !== defaultRegions.length || - currentRegions.some((r) => !defaultRegions.includes(r)); + currentRegions.length !== defaultRegionNames.length || + currentRegions.some((r) => !defaultRegionNames.includes(r)); const saveState = resolveSaveState([ [isSubmitting, { status: "saving" }], @@ -109,9 +114,9 @@ const RegionsForm: React.FC = ({ ]); const displayValue = - defaultRegions.length === 0 ? null : defaultRegions.length <= 2 ? ( + defaultRegionNames.length === 0 ? null : defaultRegionNames.length <= 2 ? ( - {defaultRegions.map((r, i) => ( + {defaultRegionNames.map((r, i) => ( {i > 0 && |} @@ -128,19 +133,19 @@ const RegionsForm: React.FC = ({ ) : ( - {defaultRegions.map((r) => ( + {defaultRegionNames.map((r) => ( ))} ); const comboboxOptions: ComboboxOption[] = unselectedRegions.map((region) => ({ - value: region, - searchValue: region, + value: region.name, + searchValue: region.name, label: (
- - {region} + + {region.name}
), })); diff --git a/web/apps/dashboard/app/(app)/[workspaceSlug]/projects/new/steps/onboarding-environment-provider.tsx b/web/apps/dashboard/app/(app)/[workspaceSlug]/projects/new/steps/onboarding-environment-provider.tsx index 04f09c6132f..bc1500c42bc 100644 --- a/web/apps/dashboard/app/(app)/[workspaceSlug]/projects/new/steps/onboarding-environment-provider.tsx +++ b/web/apps/dashboard/app/(app)/[workspaceSlug]/projects/new/steps/onboarding-environment-provider.tsx @@ -66,7 +66,7 @@ export const OnboardingEnvironmentSettingsProvider = ({ // Later they can change it in the settings. function useInitializeSettings( settings: EnvironmentSettings | undefined, - availableRegions: string[] | undefined, + availableRegions: { id: string; name: string }[] | undefined, isActive: boolean, ) { const hasInitializedRef = useRef(false); @@ -102,10 +102,8 @@ function useInitializeSettings( if (!draft.shutdownSignal) { draft.shutdownSignal = "SIGTERM"; } - if (Object.keys(draft.regionConfig).length === 0) { - for (const region of availableRegions) { - draft.regionConfig[region] = 1; - } + if (draft.regions.length === 0) { + draft.regions = availableRegions.map((r) => ({ id: r.id, name: r.name, replicas: 1 })); } }, ); diff --git a/web/apps/dashboard/lib/collections/deploy/environment-settings.ts b/web/apps/dashboard/lib/collections/deploy/environment-settings.ts index a8a2380e1ff..c00a81ba378 100644 --- a/web/apps/dashboard/lib/collections/deploy/environment-settings.ts +++ b/web/apps/dashboard/lib/collections/deploy/environment-settings.ts @@ -42,7 +42,7 @@ const schema = z.object({ memoryMib: z.number().int(), command: z.array(z.string()), healthcheck: healthcheckSchema, - regionConfig: z.record(z.string(), z.number()), + regions: z.array(z.object({ id: z.string(), name: z.string(), replicas: z.number().int() })), shutdownSignal: z.string(), sentinelConfig: sentinelConfigSchema, }); @@ -80,7 +80,7 @@ export const environmentSettings = createCollection environmentId, }); - return [flattenSettingsResponse(environmentId, result.buildSettings, result.runtimeSettings)]; + return [flattenSettingsResponse(environmentId, result.buildSettings, result.runtimeSettings, result.regionalSettings)]; }, getKey: (item) => item.environmentId, id: "environmentSettings", @@ -108,6 +108,7 @@ function flattenSettingsResponse( environmentId: string, build: SettingsResponse["buildSettings"], runtime: SettingsResponse["runtimeSettings"], + regional: SettingsResponse["regionalSettings"], ): EnvironmentSettings { return { environmentId, @@ -118,7 +119,7 @@ function flattenSettingsResponse( memoryMib: runtime?.memoryMib ?? 256, command: runtime?.command ?? [], healthcheck: runtime?.healthcheck ?? null, - regionConfig: runtime?.regionConfig ?? {}, + regions: regional.map((r) => ({ id: r.regionId, name: r.regionId, replicas: r.replicas })), shutdownSignal: "SIGTERM", sentinelConfig: runtime?.sentinelConfig, }; @@ -200,32 +201,28 @@ export function buildSettingsMutations( ); } - const origRegions = Object.keys(original.regionConfig).sort(); - const modRegions = Object.keys(modified.regionConfig).sort(); - const regionsChanged = changed(origRegions, modRegions); + const origRegionIds = original.regions.map((r) => r.id).sort(); + const modRegionIds = modified.regions.map((r) => r.id).sort(); + const regionsChanged = changed(origRegionIds, modRegionIds); if (regionsChanged) { mutations.push( trpcClient.deploy.environmentSettings.runtime.updateRegions.mutate({ environmentId, - regions: modRegions, + regionIds: modRegionIds, }), ); } - const origValues = Object.values(original.regionConfig); - const modValues = Object.values(modified.regionConfig); - const instancesChanged = - !regionsChanged && - origValues.length === modValues.length && - modValues.length > 0 && - modValues[0] !== origValues[0]; + const origReplicas = original.regions.at(0)?.replicas ?? 1; + const modReplicas = modified.regions.at(0)?.replicas ?? 1; + const instancesChanged = !regionsChanged && modified.regions.length > 0 && modReplicas !== origReplicas; if (instancesChanged) { mutations.push( trpcClient.deploy.environmentSettings.runtime.updateInstances.mutate({ environmentId, - replicasPerRegion: modValues[0], + replicasPerRegion: modReplicas, }), ); } diff --git a/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/get-available-regions.ts b/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/get-available-regions.ts index 16337bf3be6..d08d24fc137 100644 --- a/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/get-available-regions.ts +++ b/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/get-available-regions.ts @@ -3,7 +3,7 @@ import { workspaceProcedure } from "../../../trpc"; export const getAvailableRegions = workspaceProcedure.query(async () => { const regions = await db.query.clusterRegions.findMany({ - columns: { name: true }, + columns: { id: true, name: true }, }); - return regions.map((r) => r.name); + return regions; }); diff --git a/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/get.ts b/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/get.ts index 56855bdcbb4..c8b24292c71 100644 --- a/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/get.ts +++ b/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/get.ts @@ -1,5 +1,5 @@ import { and, db, eq } from "@/lib/db"; -import { appBuildSettings, appRuntimeSettings } from "@unkey/db/src/schema"; +import { appBuildSettings, appRegionalSettings, appRuntimeSettings } from "@unkey/db/src/schema"; import { z } from "zod"; import { workspaceProcedure } from "../../../trpc"; import type { SentinelConfig } from "./sentinel/update-middleware"; @@ -7,7 +7,7 @@ import type { SentinelConfig } from "./sentinel/update-middleware"; export const getEnvironmentSettings = workspaceProcedure .input(z.object({ environmentId: z.string() })) .query(async ({ ctx, input }) => { - const [buildSettings, runtimeSettings] = await Promise.all([ + const [buildSettings, runtimeSettings, regionalSettings] = await Promise.all([ db.query.appBuildSettings.findFirst({ where: and( eq(appBuildSettings.workspaceId, ctx.workspace.id), @@ -20,6 +20,12 @@ export const getEnvironmentSettings = workspaceProcedure eq(appRuntimeSettings.environmentId, input.environmentId), ), }), + db.query.appRegionalSettings.findMany({ + where: and( + eq(appRegionalSettings.workspaceId, ctx.workspace.id), + eq(appRegionalSettings.environmentId, input.environmentId), + ), + }), ]); return { @@ -35,5 +41,6 @@ export const getEnvironmentSettings = workspaceProcedure : undefined, } : null, + regionalSettings, }; }); diff --git a/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/runtime/update-regions.ts b/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/runtime/update-regions.ts index c24748b32c6..da55d8400f2 100644 --- a/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/runtime/update-regions.ts +++ b/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/runtime/update-regions.ts @@ -1,5 +1,5 @@ import { and, db, eq, inArray, notInArray } from "@/lib/db"; -import { appRegionalSettings, clusterRegions, environments } from "@unkey/db/src/schema"; +import { appRegionalSettings, environments } from "@unkey/db/src/schema"; import { z } from "zod"; import { TRPCError } from "@trpc/server"; import { workspaceProcedure } from "../../../../trpc"; @@ -9,28 +9,12 @@ export const updateRegions = workspaceProcedure .input( z.object({ environmentId: z.string(), - regions: z.array(z.string()).min(1), + regionIds: z.array(z.string()).min(1), }), ) .mutation(async ({ ctx, input }) => { const envIds = await resolveProjectEnvironmentIds(ctx.workspace.id, input.environmentId); - // Resolve region names to IDs - const regions = await db.query.clusterRegions.findMany({ - where: inArray(clusterRegions.name, input.regions), - columns: { id: true, name: true }, - }); - const regionIds = regions.map((r) => r.id); - - if (regionIds.length !== input.regions.length) { - const found = new Set(regions.map((r) => r.name)); - const missing = input.regions.filter((r) => !found.has(r)); - throw new TRPCError({ - code: "BAD_REQUEST", - message: `Unknown regions: ${missing.join(", ")}`, - }); - } - // Get appId from the environment const env = await db.query.environments.findFirst({ where: and( @@ -43,7 +27,7 @@ export const updateRegions = workspaceProcedure throw new TRPCError({ code: "NOT_FOUND", message: "Environment not found" }); } - // Get existing scaling settings to preserve values for new regions + // Get existing regional settings to preserve replica counts for existing regions const existingSettings = await db.query.appRegionalSettings.findMany({ where: and( eq(appRegionalSettings.workspaceId, ctx.workspace.id), @@ -61,12 +45,12 @@ export const updateRegions = workspaceProcedure and( eq(appRegionalSettings.workspaceId, ctx.workspace.id), eq(appRegionalSettings.environmentId, envId), - notInArray(appRegionalSettings.regionId, regionIds), + notInArray(appRegionalSettings.regionId, input.regionIds), ), ); // Insert rows for newly added regions - for (const regionId of regionIds) { + for (const regionId of input.regionIds) { const existing = existingSettings.find((s) => s.regionId === regionId); if (!existing) { await db.insert(appRegionalSettings).values({ From 76f1c16bacf13ede56e85f959ba0f475ddf00647 Mon Sep 17 00:00:00 2001 From: chronark Date: Sat, 7 Mar 2026 12:57:17 +0100 Subject: [PATCH 11/24] fix: remove AVAILABLE_REGIONS from .env.example --- web/apps/dashboard/.env.example | 2 -- 1 file changed, 2 deletions(-) diff --git a/web/apps/dashboard/.env.example b/web/apps/dashboard/.env.example index 4399362debd..491b5d451f5 100644 --- a/web/apps/dashboard/.env.example +++ b/web/apps/dashboard/.env.example @@ -29,7 +29,5 @@ VAULT_TOKEN=vault-test-token-123 CLICKHOUSE_URL= # Optional -AVAILABLE_REGIONS=us-east-1 - TRIGGER_API_KEY= NEXT_PUBLIC_TRIGGER_PUBLIC_KEY= From 21e23d2eca7af306bf231dde14050ddd3c5b86ee Mon Sep 17 00:00:00 2001 From: chronark Date: Sat, 7 Mar 2026 12:58:30 +0100 Subject: [PATCH 12/24] feat: add migration script for region_config to app_regional_settings --- .../region-config-to-regional-settings.ts | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 web/tools/migrate/region-config-to-regional-settings.ts diff --git a/web/tools/migrate/region-config-to-regional-settings.ts b/web/tools/migrate/region-config-to-regional-settings.ts new file mode 100644 index 00000000000..9413ce46711 --- /dev/null +++ b/web/tools/migrate/region-config-to-regional-settings.ts @@ -0,0 +1,76 @@ +import { mysqlDrizzle, schema } from "@unkey/db"; +import mysql from "mysql2/promise"; + +/** + * Migrates region_config JSON from app_runtime_settings into + * individual app_regional_settings rows. + * + * For each app_runtime_settings row with a non-empty region_config, + * creates one app_regional_settings row per region entry. + * + * Safe to run multiple times — uses INSERT IGNORE to skip existing rows. + */ +async function main() { + const conn = await mysql.createConnection( + `mysql://${process.env.DATABASE_USERNAME}:${process.env.DATABASE_PASSWORD}@${process.env.DATABASE_HOST}:3306/unkey?ssl={}`, + ); + + await conn.ping(); + const db = mysqlDrizzle(conn, { schema, mode: "default" }); + + const runtimeSettings = await db.query.appRuntimeSettings.findMany({ + columns: { + workspaceId: true, + appId: true, + environmentId: true, + regionConfig: true, + }, + }); + + console.info(`Found ${runtimeSettings.length} runtime settings rows`); + + let created = 0; + let skipped = 0; + + for (const row of runtimeSettings) { + const regionConfig = (row.regionConfig ?? {}) as Record; + const regions = Object.entries(regionConfig); + + if (regions.length === 0) { + skipped++; + continue; + } + + for (const [regionId, replicas] of regions) { + try { + await db.insert(schema.appRegionalSettings).values({ + workspaceId: row.workspaceId, + appId: row.appId, + environmentId: row.environmentId, + regionId, + replicas, + createdAt: Date.now(), + updatedAt: Date.now(), + }); + created++; + console.info( + `Created: app=${row.appId} env=${row.environmentId} region=${regionId} replicas=${replicas}`, + ); + } catch (err: unknown) { + if (err instanceof Error && err.message.includes("Duplicate entry")) { + console.info( + `Skipped (already exists): app=${row.appId} env=${row.environmentId} region=${regionId}`, + ); + skipped++; + } else { + throw err; + } + } + } + } + + console.info(`Migration complete. Created: ${created}, Skipped: ${skipped}`); + await conn.end(); +} + +main(); From 1723b632d5ac74b61d2c1f0e3ef1d06f9290556d Mon Sep 17 00:00:00 2001 From: chronark Date: Sat, 7 Mar 2026 13:09:05 +0100 Subject: [PATCH 13/24] feat: add Heartbeat RPC to ClusterService proto --- gen/proto/ctrl/v1/cluster.pb.go | 156 +++++++++++++++--- .../ctrl/v1/ctrlv1connect/cluster.connect.go | 35 ++++ gen/rpc/ctrl/cluster_generated.go | 14 ++ svc/ctrl/proto/ctrl/v1/cluster.proto | 18 ++ .../internal/testutil/mock_cluster_client.go | 8 + .../dashboard/gen/proto/ctrl/v1/cluster_pb.ts | 57 ++++++- .../deploy/environment-settings.ts | 12 +- .../runtime/update-regions.ts | 20 ++- .../db/src/schema/app_regional_settings.ts | 48 ++---- .../schema/horizontal_autoscaling_policies.ts | 26 ++- 10 files changed, 307 insertions(+), 87 deletions(-) diff --git a/gen/proto/ctrl/v1/cluster.pb.go b/gen/proto/ctrl/v1/cluster.pb.go index 0091102f3c8..021d77df9ec 100644 --- a/gen/proto/ctrl/v1/cluster.pb.go +++ b/gen/proto/ctrl/v1/cluster.pb.go @@ -1499,6 +1499,99 @@ func (x *DeleteDeployment) GetK8SName() string { return "" } +// HeartbeatRequest is sent periodically by krane agents to register their +// presence. The control plane uses this to populate cluster_regions and +// clusters tables. +type HeartbeatRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // region is the geographic region name (e.g. "us-east-1"). + Region string `protobuf:"bytes,1,opt,name=region,proto3" json:"region,omitempty"` + // platform is the infrastructure provider (e.g. "aws", "gcp", "local"). + Platform string `protobuf:"bytes,2,opt,name=platform,proto3" json:"platform,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *HeartbeatRequest) Reset() { + *x = HeartbeatRequest{} + mi := &file_ctrl_v1_cluster_proto_msgTypes[19] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *HeartbeatRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*HeartbeatRequest) ProtoMessage() {} + +func (x *HeartbeatRequest) ProtoReflect() protoreflect.Message { + mi := &file_ctrl_v1_cluster_proto_msgTypes[19] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use HeartbeatRequest.ProtoReflect.Descriptor instead. +func (*HeartbeatRequest) Descriptor() ([]byte, []int) { + return file_ctrl_v1_cluster_proto_rawDescGZIP(), []int{19} +} + +func (x *HeartbeatRequest) GetRegion() string { + if x != nil { + return x.Region + } + return "" +} + +func (x *HeartbeatRequest) GetPlatform() string { + if x != nil { + return x.Platform + } + return "" +} + +type HeartbeatResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *HeartbeatResponse) Reset() { + *x = HeartbeatResponse{} + mi := &file_ctrl_v1_cluster_proto_msgTypes[20] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *HeartbeatResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*HeartbeatResponse) ProtoMessage() {} + +func (x *HeartbeatResponse) ProtoReflect() protoreflect.Message { + mi := &file_ctrl_v1_cluster_proto_msgTypes[20] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use HeartbeatResponse.ProtoReflect.Descriptor instead. +func (*HeartbeatResponse) Descriptor() ([]byte, []int) { + return file_ctrl_v1_cluster_proto_rawDescGZIP(), []int{20} +} + type ReportDeploymentStatusRequest_Update struct { state protoimpl.MessageState `protogen:"open.v1"` K8SName string `protobuf:"bytes,1,opt,name=k8s_name,json=k8sName,proto3" json:"k8s_name,omitempty"` @@ -1509,7 +1602,7 @@ type ReportDeploymentStatusRequest_Update struct { func (x *ReportDeploymentStatusRequest_Update) Reset() { *x = ReportDeploymentStatusRequest_Update{} - mi := &file_ctrl_v1_cluster_proto_msgTypes[19] + mi := &file_ctrl_v1_cluster_proto_msgTypes[21] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1521,7 +1614,7 @@ func (x *ReportDeploymentStatusRequest_Update) String() string { func (*ReportDeploymentStatusRequest_Update) ProtoMessage() {} func (x *ReportDeploymentStatusRequest_Update) ProtoReflect() protoreflect.Message { - mi := &file_ctrl_v1_cluster_proto_msgTypes[19] + mi := &file_ctrl_v1_cluster_proto_msgTypes[21] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1560,7 +1653,7 @@ type ReportDeploymentStatusRequest_Delete struct { func (x *ReportDeploymentStatusRequest_Delete) Reset() { *x = ReportDeploymentStatusRequest_Delete{} - mi := &file_ctrl_v1_cluster_proto_msgTypes[20] + mi := &file_ctrl_v1_cluster_proto_msgTypes[22] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1572,7 +1665,7 @@ func (x *ReportDeploymentStatusRequest_Delete) String() string { func (*ReportDeploymentStatusRequest_Delete) ProtoMessage() {} func (x *ReportDeploymentStatusRequest_Delete) ProtoReflect() protoreflect.Message { - mi := &file_ctrl_v1_cluster_proto_msgTypes[20] + mi := &file_ctrl_v1_cluster_proto_msgTypes[22] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1608,7 +1701,7 @@ type ReportDeploymentStatusRequest_Update_Instance struct { func (x *ReportDeploymentStatusRequest_Update_Instance) Reset() { *x = ReportDeploymentStatusRequest_Update_Instance{} - mi := &file_ctrl_v1_cluster_proto_msgTypes[21] + mi := &file_ctrl_v1_cluster_proto_msgTypes[23] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1620,7 +1713,7 @@ func (x *ReportDeploymentStatusRequest_Update_Instance) String() string { func (*ReportDeploymentStatusRequest_Update_Instance) ProtoMessage() {} func (x *ReportDeploymentStatusRequest_Update_Instance) ProtoReflect() protoreflect.Message { - mi := &file_ctrl_v1_cluster_proto_msgTypes[21] + mi := &file_ctrl_v1_cluster_proto_msgTypes[23] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1782,12 +1875,16 @@ const file_ctrl_v1_cluster_proto_rawDesc = "" + "\f_healthcheck\"R\n" + "\x10DeleteDeployment\x12#\n" + "\rk8s_namespace\x18\x01 \x01(\tR\fk8sNamespace\x12\x19\n" + - "\bk8s_name\x18\x02 \x01(\tR\ak8sName*]\n" + + "\bk8s_name\x18\x02 \x01(\tR\ak8sName\"F\n" + + "\x10HeartbeatRequest\x12\x16\n" + + "\x06region\x18\x01 \x01(\tR\x06region\x12\x1a\n" + + "\bplatform\x18\x02 \x01(\tR\bplatform\"\x13\n" + + "\x11HeartbeatResponse*]\n" + "\x06Health\x12\x16\n" + "\x12HEALTH_UNSPECIFIED\x10\x00\x12\x12\n" + "\x0eHEALTH_HEALTHY\x10\x01\x12\x14\n" + "\x10HEALTH_UNHEALTHY\x10\x02\x12\x11\n" + - "\rHEALTH_PAUSED\x10\x032\xa8\x06\n" + + "\rHEALTH_PAUSED\x10\x032\xec\x06\n" + "\x0eClusterService\x12P\n" + "\x10WatchDeployments\x12 .ctrl.v1.WatchDeploymentsRequest\x1a\x18.ctrl.v1.DeploymentState0\x01\x12J\n" + "\x0eWatchSentinels\x12\x1e.ctrl.v1.WatchSentinelsRequest\x1a\x16.ctrl.v1.SentinelState0\x01\x12Z\n" + @@ -1796,7 +1893,8 @@ const file_ctrl_v1_cluster_proto_rawDesc = "" + "\x19GetDesiredDeploymentState\x12).ctrl.v1.GetDesiredDeploymentStateRequest\x1a\x18.ctrl.v1.DeploymentState\x12i\n" + "\x16ReportDeploymentStatus\x12&.ctrl.v1.ReportDeploymentStatusRequest\x1a'.ctrl.v1.ReportDeploymentStatusResponse\x12m\n" + "\x1aWatchCiliumNetworkPolicies\x12*.ctrl.v1.WatchCiliumNetworkPoliciesRequest\x1a!.ctrl.v1.CiliumNetworkPolicyState0\x01\x12{\n" + - "\"GetDesiredCiliumNetworkPolicyState\x122.ctrl.v1.GetDesiredCiliumNetworkPolicyStateRequest\x1a!.ctrl.v1.CiliumNetworkPolicyStateB\x8b\x01\n" + + "\"GetDesiredCiliumNetworkPolicyState\x122.ctrl.v1.GetDesiredCiliumNetworkPolicyStateRequest\x1a!.ctrl.v1.CiliumNetworkPolicyState\x12B\n" + + "\tHeartbeat\x12\x19.ctrl.v1.HeartbeatRequest\x1a\x1a.ctrl.v1.HeartbeatResponseB\x8b\x01\n" + "\vcom.ctrl.v1B\fClusterProtoP\x01Z1github.com/unkeyed/unkey/gen/proto/ctrl/v1;ctrlv1\xa2\x02\x03CXX\xaa\x02\aCtrl.V1\xca\x02\aCtrl\\V1\xe2\x02\x13Ctrl\\V1\\GPBMetadata\xea\x02\bCtrl::V1b\x06proto3" var ( @@ -1812,7 +1910,7 @@ func file_ctrl_v1_cluster_proto_rawDescGZIP() []byte { } var file_ctrl_v1_cluster_proto_enumTypes = make([]protoimpl.EnumInfo, 2) -var file_ctrl_v1_cluster_proto_msgTypes = make([]protoimpl.MessageInfo, 22) +var file_ctrl_v1_cluster_proto_msgTypes = make([]protoimpl.MessageInfo, 24) var file_ctrl_v1_cluster_proto_goTypes = []any{ (Health)(0), // 0: ctrl.v1.Health (ReportDeploymentStatusRequest_Update_Instance_Status)(0), // 1: ctrl.v1.ReportDeploymentStatusRequest.Update.Instance.Status @@ -1835,21 +1933,23 @@ var file_ctrl_v1_cluster_proto_goTypes = []any{ (*DeleteSentinel)(nil), // 18: ctrl.v1.DeleteSentinel (*ApplyDeployment)(nil), // 19: ctrl.v1.ApplyDeployment (*DeleteDeployment)(nil), // 20: ctrl.v1.DeleteDeployment - (*ReportDeploymentStatusRequest_Update)(nil), // 21: ctrl.v1.ReportDeploymentStatusRequest.Update - (*ReportDeploymentStatusRequest_Delete)(nil), // 22: ctrl.v1.ReportDeploymentStatusRequest.Delete - (*ReportDeploymentStatusRequest_Update_Instance)(nil), // 23: ctrl.v1.ReportDeploymentStatusRequest.Update.Instance + (*HeartbeatRequest)(nil), // 21: ctrl.v1.HeartbeatRequest + (*HeartbeatResponse)(nil), // 22: ctrl.v1.HeartbeatResponse + (*ReportDeploymentStatusRequest_Update)(nil), // 23: ctrl.v1.ReportDeploymentStatusRequest.Update + (*ReportDeploymentStatusRequest_Delete)(nil), // 24: ctrl.v1.ReportDeploymentStatusRequest.Delete + (*ReportDeploymentStatusRequest_Update_Instance)(nil), // 25: ctrl.v1.ReportDeploymentStatusRequest.Update.Instance } var file_ctrl_v1_cluster_proto_depIdxs = []int32{ 3, // 0: ctrl.v1.CiliumNetworkPolicyState.apply:type_name -> ctrl.v1.ApplyCiliumNetworkPolicy 4, // 1: ctrl.v1.CiliumNetworkPolicyState.delete:type_name -> ctrl.v1.DeleteCiliumNetworkPolicy - 21, // 2: ctrl.v1.ReportDeploymentStatusRequest.update:type_name -> ctrl.v1.ReportDeploymentStatusRequest.Update - 22, // 3: ctrl.v1.ReportDeploymentStatusRequest.delete:type_name -> ctrl.v1.ReportDeploymentStatusRequest.Delete + 23, // 2: ctrl.v1.ReportDeploymentStatusRequest.update:type_name -> ctrl.v1.ReportDeploymentStatusRequest.Update + 24, // 3: ctrl.v1.ReportDeploymentStatusRequest.delete:type_name -> ctrl.v1.ReportDeploymentStatusRequest.Delete 0, // 4: ctrl.v1.ReportSentinelStatusRequest.health:type_name -> ctrl.v1.Health 17, // 5: ctrl.v1.SentinelState.apply:type_name -> ctrl.v1.ApplySentinel 18, // 6: ctrl.v1.SentinelState.delete:type_name -> ctrl.v1.DeleteSentinel 19, // 7: ctrl.v1.DeploymentState.apply:type_name -> ctrl.v1.ApplyDeployment 20, // 8: ctrl.v1.DeploymentState.delete:type_name -> ctrl.v1.DeleteDeployment - 23, // 9: ctrl.v1.ReportDeploymentStatusRequest.Update.instances:type_name -> ctrl.v1.ReportDeploymentStatusRequest.Update.Instance + 25, // 9: ctrl.v1.ReportDeploymentStatusRequest.Update.instances:type_name -> ctrl.v1.ReportDeploymentStatusRequest.Update.Instance 1, // 10: ctrl.v1.ReportDeploymentStatusRequest.Update.Instance.status:type_name -> ctrl.v1.ReportDeploymentStatusRequest.Update.Instance.Status 13, // 11: ctrl.v1.ClusterService.WatchDeployments:input_type -> ctrl.v1.WatchDeploymentsRequest 14, // 12: ctrl.v1.ClusterService.WatchSentinels:input_type -> ctrl.v1.WatchSentinelsRequest @@ -1859,16 +1959,18 @@ var file_ctrl_v1_cluster_proto_depIdxs = []int32{ 9, // 16: ctrl.v1.ClusterService.ReportDeploymentStatus:input_type -> ctrl.v1.ReportDeploymentStatusRequest 2, // 17: ctrl.v1.ClusterService.WatchCiliumNetworkPolicies:input_type -> ctrl.v1.WatchCiliumNetworkPoliciesRequest 6, // 18: ctrl.v1.ClusterService.GetDesiredCiliumNetworkPolicyState:input_type -> ctrl.v1.GetDesiredCiliumNetworkPolicyStateRequest - 16, // 19: ctrl.v1.ClusterService.WatchDeployments:output_type -> ctrl.v1.DeploymentState - 15, // 20: ctrl.v1.ClusterService.WatchSentinels:output_type -> ctrl.v1.SentinelState - 15, // 21: ctrl.v1.ClusterService.GetDesiredSentinelState:output_type -> ctrl.v1.SentinelState - 12, // 22: ctrl.v1.ClusterService.ReportSentinelStatus:output_type -> ctrl.v1.ReportSentinelStatusResponse - 16, // 23: ctrl.v1.ClusterService.GetDesiredDeploymentState:output_type -> ctrl.v1.DeploymentState - 10, // 24: ctrl.v1.ClusterService.ReportDeploymentStatus:output_type -> ctrl.v1.ReportDeploymentStatusResponse - 5, // 25: ctrl.v1.ClusterService.WatchCiliumNetworkPolicies:output_type -> ctrl.v1.CiliumNetworkPolicyState - 5, // 26: ctrl.v1.ClusterService.GetDesiredCiliumNetworkPolicyState:output_type -> ctrl.v1.CiliumNetworkPolicyState - 19, // [19:27] is the sub-list for method output_type - 11, // [11:19] is the sub-list for method input_type + 21, // 19: ctrl.v1.ClusterService.Heartbeat:input_type -> ctrl.v1.HeartbeatRequest + 16, // 20: ctrl.v1.ClusterService.WatchDeployments:output_type -> ctrl.v1.DeploymentState + 15, // 21: ctrl.v1.ClusterService.WatchSentinels:output_type -> ctrl.v1.SentinelState + 15, // 22: ctrl.v1.ClusterService.GetDesiredSentinelState:output_type -> ctrl.v1.SentinelState + 12, // 23: ctrl.v1.ClusterService.ReportSentinelStatus:output_type -> ctrl.v1.ReportSentinelStatusResponse + 16, // 24: ctrl.v1.ClusterService.GetDesiredDeploymentState:output_type -> ctrl.v1.DeploymentState + 10, // 25: ctrl.v1.ClusterService.ReportDeploymentStatus:output_type -> ctrl.v1.ReportDeploymentStatusResponse + 5, // 26: ctrl.v1.ClusterService.WatchCiliumNetworkPolicies:output_type -> ctrl.v1.CiliumNetworkPolicyState + 5, // 27: ctrl.v1.ClusterService.GetDesiredCiliumNetworkPolicyState:output_type -> ctrl.v1.CiliumNetworkPolicyState + 22, // 28: ctrl.v1.ClusterService.Heartbeat:output_type -> ctrl.v1.HeartbeatResponse + 20, // [20:29] is the sub-list for method output_type + 11, // [11:20] is the sub-list for method input_type 11, // [11:11] is the sub-list for extension type_name 11, // [11:11] is the sub-list for extension extendee 0, // [0:11] is the sub-list for field type_name @@ -1902,7 +2004,7 @@ func file_ctrl_v1_cluster_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_ctrl_v1_cluster_proto_rawDesc), len(file_ctrl_v1_cluster_proto_rawDesc)), NumEnums: 2, - NumMessages: 22, + NumMessages: 24, NumExtensions: 0, NumServices: 1, }, diff --git a/gen/proto/ctrl/v1/ctrlv1connect/cluster.connect.go b/gen/proto/ctrl/v1/ctrlv1connect/cluster.connect.go index 1aaddb026c5..5a1e137023f 100644 --- a/gen/proto/ctrl/v1/ctrlv1connect/cluster.connect.go +++ b/gen/proto/ctrl/v1/ctrlv1connect/cluster.connect.go @@ -66,6 +66,9 @@ const ( // ClusterServiceGetDesiredCiliumNetworkPolicyStateProcedure is the fully-qualified name of the // ClusterService's GetDesiredCiliumNetworkPolicyState RPC. ClusterServiceGetDesiredCiliumNetworkPolicyStateProcedure = "/ctrl.v1.ClusterService/GetDesiredCiliumNetworkPolicyState" + // ClusterServiceHeartbeatProcedure is the fully-qualified name of the ClusterService's Heartbeat + // RPC. + ClusterServiceHeartbeatProcedure = "/ctrl.v1.ClusterService/Heartbeat" ) // ClusterServiceClient is a client for the ctrl.v1.ClusterService service. @@ -96,6 +99,10 @@ type ClusterServiceClient interface { // GetDesiredCiliumNetworkPolicyState returns the current desired state for a single Cilium policy. // Used by the resync loop to verify consistency for existing resources. GetDesiredCiliumNetworkPolicyState(context.Context, *connect.Request[v1.GetDesiredCiliumNetworkPolicyStateRequest]) (*connect.Response[v1.CiliumNetworkPolicyState], error) + // Heartbeat is called periodically by krane agents to register their cluster + // and region with the control plane. This populates the cluster_regions and + // clusters tables, making regions dynamically discoverable. + Heartbeat(context.Context, *connect.Request[v1.HeartbeatRequest]) (*connect.Response[v1.HeartbeatResponse], error) } // NewClusterServiceClient constructs a client for the ctrl.v1.ClusterService service. By default, @@ -157,6 +164,12 @@ func NewClusterServiceClient(httpClient connect.HTTPClient, baseURL string, opts connect.WithSchema(clusterServiceMethods.ByName("GetDesiredCiliumNetworkPolicyState")), connect.WithClientOptions(opts...), ), + heartbeat: connect.NewClient[v1.HeartbeatRequest, v1.HeartbeatResponse]( + httpClient, + baseURL+ClusterServiceHeartbeatProcedure, + connect.WithSchema(clusterServiceMethods.ByName("Heartbeat")), + connect.WithClientOptions(opts...), + ), } } @@ -170,6 +183,7 @@ type clusterServiceClient struct { reportDeploymentStatus *connect.Client[v1.ReportDeploymentStatusRequest, v1.ReportDeploymentStatusResponse] watchCiliumNetworkPolicies *connect.Client[v1.WatchCiliumNetworkPoliciesRequest, v1.CiliumNetworkPolicyState] getDesiredCiliumNetworkPolicyState *connect.Client[v1.GetDesiredCiliumNetworkPolicyStateRequest, v1.CiliumNetworkPolicyState] + heartbeat *connect.Client[v1.HeartbeatRequest, v1.HeartbeatResponse] } // WatchDeployments calls ctrl.v1.ClusterService.WatchDeployments. @@ -213,6 +227,11 @@ func (c *clusterServiceClient) GetDesiredCiliumNetworkPolicyState(ctx context.Co return c.getDesiredCiliumNetworkPolicyState.CallUnary(ctx, req) } +// Heartbeat calls ctrl.v1.ClusterService.Heartbeat. +func (c *clusterServiceClient) Heartbeat(ctx context.Context, req *connect.Request[v1.HeartbeatRequest]) (*connect.Response[v1.HeartbeatResponse], error) { + return c.heartbeat.CallUnary(ctx, req) +} + // ClusterServiceHandler is an implementation of the ctrl.v1.ClusterService service. type ClusterServiceHandler interface { // WatchDeployments streams deployment state changes from the control plane to agents. @@ -241,6 +260,10 @@ type ClusterServiceHandler interface { // GetDesiredCiliumNetworkPolicyState returns the current desired state for a single Cilium policy. // Used by the resync loop to verify consistency for existing resources. GetDesiredCiliumNetworkPolicyState(context.Context, *connect.Request[v1.GetDesiredCiliumNetworkPolicyStateRequest]) (*connect.Response[v1.CiliumNetworkPolicyState], error) + // Heartbeat is called periodically by krane agents to register their cluster + // and region with the control plane. This populates the cluster_regions and + // clusters tables, making regions dynamically discoverable. + Heartbeat(context.Context, *connect.Request[v1.HeartbeatRequest]) (*connect.Response[v1.HeartbeatResponse], error) } // NewClusterServiceHandler builds an HTTP handler from the service implementation. It returns the @@ -298,6 +321,12 @@ func NewClusterServiceHandler(svc ClusterServiceHandler, opts ...connect.Handler connect.WithSchema(clusterServiceMethods.ByName("GetDesiredCiliumNetworkPolicyState")), connect.WithHandlerOptions(opts...), ) + clusterServiceHeartbeatHandler := connect.NewUnaryHandler( + ClusterServiceHeartbeatProcedure, + svc.Heartbeat, + connect.WithSchema(clusterServiceMethods.ByName("Heartbeat")), + connect.WithHandlerOptions(opts...), + ) return "/ctrl.v1.ClusterService/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case ClusterServiceWatchDeploymentsProcedure: @@ -316,6 +345,8 @@ func NewClusterServiceHandler(svc ClusterServiceHandler, opts ...connect.Handler clusterServiceWatchCiliumNetworkPoliciesHandler.ServeHTTP(w, r) case ClusterServiceGetDesiredCiliumNetworkPolicyStateProcedure: clusterServiceGetDesiredCiliumNetworkPolicyStateHandler.ServeHTTP(w, r) + case ClusterServiceHeartbeatProcedure: + clusterServiceHeartbeatHandler.ServeHTTP(w, r) default: http.NotFound(w, r) } @@ -356,3 +387,7 @@ func (UnimplementedClusterServiceHandler) WatchCiliumNetworkPolicies(context.Con func (UnimplementedClusterServiceHandler) GetDesiredCiliumNetworkPolicyState(context.Context, *connect.Request[v1.GetDesiredCiliumNetworkPolicyStateRequest]) (*connect.Response[v1.CiliumNetworkPolicyState], error) { return nil, connect.NewError(connect.CodeUnimplemented, errors.New("ctrl.v1.ClusterService.GetDesiredCiliumNetworkPolicyState is not implemented")) } + +func (UnimplementedClusterServiceHandler) Heartbeat(context.Context, *connect.Request[v1.HeartbeatRequest]) (*connect.Response[v1.HeartbeatResponse], error) { + return nil, connect.NewError(connect.CodeUnimplemented, errors.New("ctrl.v1.ClusterService.Heartbeat is not implemented")) +} diff --git a/gen/rpc/ctrl/cluster_generated.go b/gen/rpc/ctrl/cluster_generated.go index 229faf9e616..267786f8ab0 100644 --- a/gen/rpc/ctrl/cluster_generated.go +++ b/gen/rpc/ctrl/cluster_generated.go @@ -22,6 +22,7 @@ type ClusterServiceClient interface { ReportDeploymentStatus(ctx context.Context, req *v1.ReportDeploymentStatusRequest) (*v1.ReportDeploymentStatusResponse, error) WatchCiliumNetworkPolicies(ctx context.Context, req *v1.WatchCiliumNetworkPoliciesRequest) (*connect.ServerStreamForClient[v1.CiliumNetworkPolicyState], error) GetDesiredCiliumNetworkPolicyState(ctx context.Context, req *v1.GetDesiredCiliumNetworkPolicyStateRequest) (*v1.CiliumNetworkPolicyState, error) + Heartbeat(ctx context.Context, req *v1.HeartbeatRequest) (*v1.HeartbeatResponse, error) } var _ ClusterServiceClient = (*ConnectClusterServiceClient)(nil) @@ -112,3 +113,16 @@ func (c *ConnectClusterServiceClient) GetDesiredCiliumNetworkPolicyState(ctx con } return resp.Msg, nil } + +func (c *ConnectClusterServiceClient) Heartbeat(ctx context.Context, req *v1.HeartbeatRequest) (*v1.HeartbeatResponse, error) { + ctx, span := tracing.Start(ctx, "ClusterService.Heartbeat") + defer span.End() + resp, err := c.inner.Heartbeat(ctx, connect.NewRequest(req)) + if err != nil { + if connect.CodeOf(err) != connect.CodeNotFound { + tracing.RecordError(span, err) + } + return nil, err + } + return resp.Msg, nil +} diff --git a/svc/ctrl/proto/ctrl/v1/cluster.proto b/svc/ctrl/proto/ctrl/v1/cluster.proto index 89c67621c12..b9cbadf1214 100644 --- a/svc/ctrl/proto/ctrl/v1/cluster.proto +++ b/svc/ctrl/proto/ctrl/v1/cluster.proto @@ -64,6 +64,11 @@ service ClusterService { // GetDesiredCiliumNetworkPolicyState returns the current desired state for a single Cilium policy. // Used by the resync loop to verify consistency for existing resources. rpc GetDesiredCiliumNetworkPolicyState(GetDesiredCiliumNetworkPolicyStateRequest) returns (CiliumNetworkPolicyState); + + // Heartbeat is called periodically by krane agents to register their cluster + // and region with the control plane. This populates the cluster_regions and + // clusters tables, making regions dynamically discoverable. + rpc Heartbeat(HeartbeatRequest) returns (HeartbeatResponse); } message WatchCiliumNetworkPoliciesRequest { @@ -323,3 +328,16 @@ message DeleteDeployment { string k8s_namespace = 1; string k8s_name = 2; } + +// HeartbeatRequest is sent periodically by krane agents to register their +// presence. The control plane uses this to populate cluster_regions and +// clusters tables. +message HeartbeatRequest { + // region is the geographic region name (e.g. "us-east-1"). + string region = 1; + + // platform is the infrastructure provider (e.g. "aws", "gcp", "local"). + string platform = 2; +} + +message HeartbeatResponse {} diff --git a/svc/krane/internal/testutil/mock_cluster_client.go b/svc/krane/internal/testutil/mock_cluster_client.go index f1db2690901..7b45983080e 100644 --- a/svc/krane/internal/testutil/mock_cluster_client.go +++ b/svc/krane/internal/testutil/mock_cluster_client.go @@ -26,6 +26,7 @@ type MockClusterClient struct { GetDesiredDeploymentStateFunc func(context.Context, *ctrlv1.GetDesiredDeploymentStateRequest) (*ctrlv1.DeploymentState, error) ReportDeploymentStatusFunc func(context.Context, *ctrlv1.ReportDeploymentStatusRequest) (*ctrlv1.ReportDeploymentStatusResponse, error) GetDesiredCiliumNetworkPolicyStateFunc func(context.Context, *ctrlv1.GetDesiredCiliumNetworkPolicyStateRequest) (*ctrlv1.CiliumNetworkPolicyState, error) + HeartbeatFunc func(context.Context, *ctrlv1.HeartbeatRequest) (*ctrlv1.HeartbeatResponse, error) ReportDeploymentStatusCalls []*ctrlv1.ReportDeploymentStatusRequest ReportSentinelStatusCalls []*ctrlv1.ReportSentinelStatusRequest } @@ -87,3 +88,10 @@ func (m *MockClusterClient) GetDesiredCiliumNetworkPolicyState(ctx context.Conte } return &ctrlv1.CiliumNetworkPolicyState{}, nil } + +func (m *MockClusterClient) Heartbeat(ctx context.Context, req *ctrlv1.HeartbeatRequest) (*ctrlv1.HeartbeatResponse, error) { + if m.HeartbeatFunc != nil { + return m.HeartbeatFunc(ctx, req) + } + return &ctrlv1.HeartbeatResponse{}, nil +} diff --git a/web/apps/dashboard/gen/proto/ctrl/v1/cluster_pb.ts b/web/apps/dashboard/gen/proto/ctrl/v1/cluster_pb.ts index f3591e01979..17f8b10517e 100644 --- a/web/apps/dashboard/gen/proto/ctrl/v1/cluster_pb.ts +++ b/web/apps/dashboard/gen/proto/ctrl/v1/cluster_pb.ts @@ -20,7 +20,7 @@ import type { Message } from "@bufbuild/protobuf"; * Describes the file ctrl/v1/cluster.proto. */ export const file_ctrl_v1_cluster: GenFile = /*@__PURE__*/ - fileDesc("ChVjdHJsL3YxL2NsdXN0ZXIucHJvdG8SB2N0cmwudjEiTgohV2F0Y2hDaWxpdW1OZXR3b3JrUG9saWNpZXNSZXF1ZXN0Eg4KBnJlZ2lvbhgBIAEoCRIZChF2ZXJzaW9uX2xhc3Rfc2VlbhgCIAEoBCJ1ChhBcHBseUNpbGl1bU5ldHdvcmtQb2xpY3kSFQoNazhzX25hbWVzcGFjZRgBIAEoCRIQCghrOHNfbmFtZRgCIAEoCRIgChhjaWxpdW1fbmV0d29ya19wb2xpY3lfaWQYAyABKAkSDgoGcG9saWN5GAQgASgMIkQKGURlbGV0ZUNpbGl1bU5ldHdvcmtQb2xpY3kSFQoNazhzX25hbWVzcGFjZRgBIAEoCRIQCghrOHNfbmFtZRgCIAEoCSKeAQoYQ2lsaXVtTmV0d29ya1BvbGljeVN0YXRlEg8KB3ZlcnNpb24YAyABKAQSMgoFYXBwbHkYASABKAsyIS5jdHJsLnYxLkFwcGx5Q2lsaXVtTmV0d29ya1BvbGljeUgAEjQKBmRlbGV0ZRgCIAEoCzIiLmN0cmwudjEuRGVsZXRlQ2lsaXVtTmV0d29ya1BvbGljeUgAQgcKBXN0YXRlIk0KKUdldERlc2lyZWRDaWxpdW1OZXR3b3JrUG9saWN5U3RhdGVSZXF1ZXN0EiAKGGNpbGl1bV9uZXR3b3JrX3BvbGljeV9pZBgBIAEoCSI1Ch5HZXREZXNpcmVkU2VudGluZWxTdGF0ZVJlcXVlc3QSEwoLc2VudGluZWxfaWQYASABKAkiOQogR2V0RGVzaXJlZERlcGxveW1lbnRTdGF0ZVJlcXVlc3QSFQoNZGVwbG95bWVudF9pZBgBIAEoCSK3BAodUmVwb3J0RGVwbG95bWVudFN0YXR1c1JlcXVlc3QSPwoGdXBkYXRlGAEgASgLMi0uY3RybC52MS5SZXBvcnREZXBsb3ltZW50U3RhdHVzUmVxdWVzdC5VcGRhdGVIABI/CgZkZWxldGUYAiABKAsyLS5jdHJsLnYxLlJlcG9ydERlcGxveW1lbnRTdGF0dXNSZXF1ZXN0LkRlbGV0ZUgAGu0CCgZVcGRhdGUSEAoIazhzX25hbWUYASABKAkSSQoJaW5zdGFuY2VzGAIgAygLMjYuY3RybC52MS5SZXBvcnREZXBsb3ltZW50U3RhdHVzUmVxdWVzdC5VcGRhdGUuSW5zdGFuY2UahQIKCEluc3RhbmNlEhAKCGs4c19uYW1lGAEgASgJEg8KB2FkZHJlc3MYAiABKAkSFgoOY3B1X21pbGxpY29yZXMYAyABKAMSEgoKbWVtb3J5X21pYhgEIAEoAxJNCgZzdGF0dXMYBSABKA4yPS5jdHJsLnYxLlJlcG9ydERlcGxveW1lbnRTdGF0dXNSZXF1ZXN0LlVwZGF0ZS5JbnN0YW5jZS5TdGF0dXMiWwoGU3RhdHVzEhYKElNUQVRVU19VTlNQRUNJRklFRBAAEhIKDlNUQVRVU19QRU5ESU5HEAESEgoOU1RBVFVTX1JVTk5JTkcQAhIRCg1TVEFUVVNfRkFJTEVEEAMaGgoGRGVsZXRlEhAKCGs4c19uYW1lGAEgASgJQggKBmNoYW5nZSIgCh5SZXBvcnREZXBsb3ltZW50U3RhdHVzUmVzcG9uc2UibAobUmVwb3J0U2VudGluZWxTdGF0dXNSZXF1ZXN0EhAKCGs4c19uYW1lGAEgASgJEhoKEmF2YWlsYWJsZV9yZXBsaWNhcxgCIAEoBRIfCgZoZWFsdGgYAyABKA4yDy5jdHJsLnYxLkhlYWx0aCIeChxSZXBvcnRTZW50aW5lbFN0YXR1c1Jlc3BvbnNlIkQKF1dhdGNoRGVwbG95bWVudHNSZXF1ZXN0Eg4KBnJlZ2lvbhgBIAEoCRIZChF2ZXJzaW9uX2xhc3Rfc2VlbhgCIAEoBCJCChVXYXRjaFNlbnRpbmVsc1JlcXVlc3QSDgoGcmVnaW9uGAEgASgJEhkKEXZlcnNpb25fbGFzdF9zZWVuGAIgASgEIn0KDVNlbnRpbmVsU3RhdGUSDwoHdmVyc2lvbhgDIAEoBBInCgVhcHBseRgBIAEoCzIWLmN0cmwudjEuQXBwbHlTZW50aW5lbEgAEikKBmRlbGV0ZRgCIAEoCzIXLmN0cmwudjEuRGVsZXRlU2VudGluZWxIAEIHCgVzdGF0ZSKDAQoPRGVwbG95bWVudFN0YXRlEg8KB3ZlcnNpb24YAyABKAQSKQoFYXBwbHkYASABKAsyGC5jdHJsLnYxLkFwcGx5RGVwbG95bWVudEgAEisKBmRlbGV0ZRgCIAEoCzIZLmN0cmwudjEuRGVsZXRlRGVwbG95bWVudEgAQgcKBXN0YXRlIsUBCg1BcHBseVNlbnRpbmVsEhAKCGs4c19uYW1lGAEgASgJEhQKDHdvcmtzcGFjZV9pZBgCIAEoCRISCgpwcm9qZWN0X2lkGAMgASgJEhYKDmVudmlyb25tZW50X2lkGAQgASgJEhMKC3NlbnRpbmVsX2lkGAUgASgJEg0KBWltYWdlGAYgASgJEhAKCHJlcGxpY2FzGAcgASgFEhYKDmNwdV9taWxsaWNvcmVzGAggASgDEhIKCm1lbW9yeV9taWIYCSABKAMiIgoORGVsZXRlU2VudGluZWwSEAoIazhzX25hbWUYASABKAkinwMKD0FwcGx5RGVwbG95bWVudBIVCg1rOHNfbmFtZXNwYWNlGAEgASgJEhAKCGs4c19uYW1lGAIgASgJEhQKDHdvcmtzcGFjZV9pZBgDIAEoCRISCgpwcm9qZWN0X2lkGAQgASgJEhYKDmVudmlyb25tZW50X2lkGAUgASgJEhUKDWRlcGxveW1lbnRfaWQYBiABKAkSDQoFaW1hZ2UYByABKAkSEAoIcmVwbGljYXMYCCABKAUSFgoOY3B1X21pbGxpY29yZXMYCSABKAMSEgoKbWVtb3J5X21pYhgKIAEoAxIVCghidWlsZF9pZBgLIAEoCUgAiAEBEicKH2VuY3J5cHRlZF9lbnZpcm9ubWVudF92YXJpYWJsZXMYDCABKAwSDwoHY29tbWFuZBgNIAMoCRIMCgRwb3J0GA4gASgFEhcKD3NodXRkb3duX3NpZ25hbBgPIAEoCRIYCgtoZWFsdGhjaGVjaxgRIAEoDEgBiAEBEg4KBmFwcF9pZBgSIAEoCUILCglfYnVpbGRfaWRCDgoMX2hlYWx0aGNoZWNrIjsKEERlbGV0ZURlcGxveW1lbnQSFQoNazhzX25hbWVzcGFjZRgBIAEoCRIQCghrOHNfbmFtZRgCIAEoCSpdCgZIZWFsdGgSFgoSSEVBTFRIX1VOU1BFQ0lGSUVEEAASEgoOSEVBTFRIX0hFQUxUSFkQARIUChBIRUFMVEhfVU5IRUFMVEhZEAISEQoNSEVBTFRIX1BBVVNFRBADMqgGCg5DbHVzdGVyU2VydmljZRJQChBXYXRjaERlcGxveW1lbnRzEiAuY3RybC52MS5XYXRjaERlcGxveW1lbnRzUmVxdWVzdBoYLmN0cmwudjEuRGVwbG95bWVudFN0YXRlMAESSgoOV2F0Y2hTZW50aW5lbHMSHi5jdHJsLnYxLldhdGNoU2VudGluZWxzUmVxdWVzdBoWLmN0cmwudjEuU2VudGluZWxTdGF0ZTABEloKF0dldERlc2lyZWRTZW50aW5lbFN0YXRlEicuY3RybC52MS5HZXREZXNpcmVkU2VudGluZWxTdGF0ZVJlcXVlc3QaFi5jdHJsLnYxLlNlbnRpbmVsU3RhdGUSYwoUUmVwb3J0U2VudGluZWxTdGF0dXMSJC5jdHJsLnYxLlJlcG9ydFNlbnRpbmVsU3RhdHVzUmVxdWVzdBolLmN0cmwudjEuUmVwb3J0U2VudGluZWxTdGF0dXNSZXNwb25zZRJgChlHZXREZXNpcmVkRGVwbG95bWVudFN0YXRlEikuY3RybC52MS5HZXREZXNpcmVkRGVwbG95bWVudFN0YXRlUmVxdWVzdBoYLmN0cmwudjEuRGVwbG95bWVudFN0YXRlEmkKFlJlcG9ydERlcGxveW1lbnRTdGF0dXMSJi5jdHJsLnYxLlJlcG9ydERlcGxveW1lbnRTdGF0dXNSZXF1ZXN0GicuY3RybC52MS5SZXBvcnREZXBsb3ltZW50U3RhdHVzUmVzcG9uc2USbQoaV2F0Y2hDaWxpdW1OZXR3b3JrUG9saWNpZXMSKi5jdHJsLnYxLldhdGNoQ2lsaXVtTmV0d29ya1BvbGljaWVzUmVxdWVzdBohLmN0cmwudjEuQ2lsaXVtTmV0d29ya1BvbGljeVN0YXRlMAESewoiR2V0RGVzaXJlZENpbGl1bU5ldHdvcmtQb2xpY3lTdGF0ZRIyLmN0cmwudjEuR2V0RGVzaXJlZENpbGl1bU5ldHdvcmtQb2xpY3lTdGF0ZVJlcXVlc3QaIS5jdHJsLnYxLkNpbGl1bU5ldHdvcmtQb2xpY3lTdGF0ZUKLAQoLY29tLmN0cmwudjFCDENsdXN0ZXJQcm90b1ABWjFnaXRodWIuY29tL3Vua2V5ZWQvdW5rZXkvZ2VuL3Byb3RvL2N0cmwvdjE7Y3RybHYxogIDQ1hYqgIHQ3RybC5WMcoCB0N0cmxcVjHiAhNDdHJsXFYxXEdQQk1ldGFkYXRh6gIIQ3RybDo6VjFiBnByb3RvMw"); + fileDesc("ChVjdHJsL3YxL2NsdXN0ZXIucHJvdG8SB2N0cmwudjEiTgohV2F0Y2hDaWxpdW1OZXR3b3JrUG9saWNpZXNSZXF1ZXN0Eg4KBnJlZ2lvbhgBIAEoCRIZChF2ZXJzaW9uX2xhc3Rfc2VlbhgCIAEoBCJ1ChhBcHBseUNpbGl1bU5ldHdvcmtQb2xpY3kSFQoNazhzX25hbWVzcGFjZRgBIAEoCRIQCghrOHNfbmFtZRgCIAEoCRIgChhjaWxpdW1fbmV0d29ya19wb2xpY3lfaWQYAyABKAkSDgoGcG9saWN5GAQgASgMIkQKGURlbGV0ZUNpbGl1bU5ldHdvcmtQb2xpY3kSFQoNazhzX25hbWVzcGFjZRgBIAEoCRIQCghrOHNfbmFtZRgCIAEoCSKeAQoYQ2lsaXVtTmV0d29ya1BvbGljeVN0YXRlEg8KB3ZlcnNpb24YAyABKAQSMgoFYXBwbHkYASABKAsyIS5jdHJsLnYxLkFwcGx5Q2lsaXVtTmV0d29ya1BvbGljeUgAEjQKBmRlbGV0ZRgCIAEoCzIiLmN0cmwudjEuRGVsZXRlQ2lsaXVtTmV0d29ya1BvbGljeUgAQgcKBXN0YXRlIk0KKUdldERlc2lyZWRDaWxpdW1OZXR3b3JrUG9saWN5U3RhdGVSZXF1ZXN0EiAKGGNpbGl1bV9uZXR3b3JrX3BvbGljeV9pZBgBIAEoCSI1Ch5HZXREZXNpcmVkU2VudGluZWxTdGF0ZVJlcXVlc3QSEwoLc2VudGluZWxfaWQYASABKAkiOQogR2V0RGVzaXJlZERlcGxveW1lbnRTdGF0ZVJlcXVlc3QSFQoNZGVwbG95bWVudF9pZBgBIAEoCSK3BAodUmVwb3J0RGVwbG95bWVudFN0YXR1c1JlcXVlc3QSPwoGdXBkYXRlGAEgASgLMi0uY3RybC52MS5SZXBvcnREZXBsb3ltZW50U3RhdHVzUmVxdWVzdC5VcGRhdGVIABI/CgZkZWxldGUYAiABKAsyLS5jdHJsLnYxLlJlcG9ydERlcGxveW1lbnRTdGF0dXNSZXF1ZXN0LkRlbGV0ZUgAGu0CCgZVcGRhdGUSEAoIazhzX25hbWUYASABKAkSSQoJaW5zdGFuY2VzGAIgAygLMjYuY3RybC52MS5SZXBvcnREZXBsb3ltZW50U3RhdHVzUmVxdWVzdC5VcGRhdGUuSW5zdGFuY2UahQIKCEluc3RhbmNlEhAKCGs4c19uYW1lGAEgASgJEg8KB2FkZHJlc3MYAiABKAkSFgoOY3B1X21pbGxpY29yZXMYAyABKAMSEgoKbWVtb3J5X21pYhgEIAEoAxJNCgZzdGF0dXMYBSABKA4yPS5jdHJsLnYxLlJlcG9ydERlcGxveW1lbnRTdGF0dXNSZXF1ZXN0LlVwZGF0ZS5JbnN0YW5jZS5TdGF0dXMiWwoGU3RhdHVzEhYKElNUQVRVU19VTlNQRUNJRklFRBAAEhIKDlNUQVRVU19QRU5ESU5HEAESEgoOU1RBVFVTX1JVTk5JTkcQAhIRCg1TVEFUVVNfRkFJTEVEEAMaGgoGRGVsZXRlEhAKCGs4c19uYW1lGAEgASgJQggKBmNoYW5nZSIgCh5SZXBvcnREZXBsb3ltZW50U3RhdHVzUmVzcG9uc2UibAobUmVwb3J0U2VudGluZWxTdGF0dXNSZXF1ZXN0EhAKCGs4c19uYW1lGAEgASgJEhoKEmF2YWlsYWJsZV9yZXBsaWNhcxgCIAEoBRIfCgZoZWFsdGgYAyABKA4yDy5jdHJsLnYxLkhlYWx0aCIeChxSZXBvcnRTZW50aW5lbFN0YXR1c1Jlc3BvbnNlIkQKF1dhdGNoRGVwbG95bWVudHNSZXF1ZXN0Eg4KBnJlZ2lvbhgBIAEoCRIZChF2ZXJzaW9uX2xhc3Rfc2VlbhgCIAEoBCJCChVXYXRjaFNlbnRpbmVsc1JlcXVlc3QSDgoGcmVnaW9uGAEgASgJEhkKEXZlcnNpb25fbGFzdF9zZWVuGAIgASgEIn0KDVNlbnRpbmVsU3RhdGUSDwoHdmVyc2lvbhgDIAEoBBInCgVhcHBseRgBIAEoCzIWLmN0cmwudjEuQXBwbHlTZW50aW5lbEgAEikKBmRlbGV0ZRgCIAEoCzIXLmN0cmwudjEuRGVsZXRlU2VudGluZWxIAEIHCgVzdGF0ZSKDAQoPRGVwbG95bWVudFN0YXRlEg8KB3ZlcnNpb24YAyABKAQSKQoFYXBwbHkYASABKAsyGC5jdHJsLnYxLkFwcGx5RGVwbG95bWVudEgAEisKBmRlbGV0ZRgCIAEoCzIZLmN0cmwudjEuRGVsZXRlRGVwbG95bWVudEgAQgcKBXN0YXRlIsUBCg1BcHBseVNlbnRpbmVsEhAKCGs4c19uYW1lGAEgASgJEhQKDHdvcmtzcGFjZV9pZBgCIAEoCRISCgpwcm9qZWN0X2lkGAMgASgJEhYKDmVudmlyb25tZW50X2lkGAQgASgJEhMKC3NlbnRpbmVsX2lkGAUgASgJEg0KBWltYWdlGAYgASgJEhAKCHJlcGxpY2FzGAcgASgFEhYKDmNwdV9taWxsaWNvcmVzGAggASgDEhIKCm1lbW9yeV9taWIYCSABKAMiIgoORGVsZXRlU2VudGluZWwSEAoIazhzX25hbWUYASABKAkinwMKD0FwcGx5RGVwbG95bWVudBIVCg1rOHNfbmFtZXNwYWNlGAEgASgJEhAKCGs4c19uYW1lGAIgASgJEhQKDHdvcmtzcGFjZV9pZBgDIAEoCRISCgpwcm9qZWN0X2lkGAQgASgJEhYKDmVudmlyb25tZW50X2lkGAUgASgJEhUKDWRlcGxveW1lbnRfaWQYBiABKAkSDQoFaW1hZ2UYByABKAkSEAoIcmVwbGljYXMYCCABKAUSFgoOY3B1X21pbGxpY29yZXMYCSABKAMSEgoKbWVtb3J5X21pYhgKIAEoAxIVCghidWlsZF9pZBgLIAEoCUgAiAEBEicKH2VuY3J5cHRlZF9lbnZpcm9ubWVudF92YXJpYWJsZXMYDCABKAwSDwoHY29tbWFuZBgNIAMoCRIMCgRwb3J0GA4gASgFEhcKD3NodXRkb3duX3NpZ25hbBgPIAEoCRIYCgtoZWFsdGhjaGVjaxgRIAEoDEgBiAEBEg4KBmFwcF9pZBgSIAEoCUILCglfYnVpbGRfaWRCDgoMX2hlYWx0aGNoZWNrIjsKEERlbGV0ZURlcGxveW1lbnQSFQoNazhzX25hbWVzcGFjZRgBIAEoCRIQCghrOHNfbmFtZRgCIAEoCSI0ChBIZWFydGJlYXRSZXF1ZXN0Eg4KBnJlZ2lvbhgBIAEoCRIQCghwbGF0Zm9ybRgCIAEoCSITChFIZWFydGJlYXRSZXNwb25zZSpdCgZIZWFsdGgSFgoSSEVBTFRIX1VOU1BFQ0lGSUVEEAASEgoOSEVBTFRIX0hFQUxUSFkQARIUChBIRUFMVEhfVU5IRUFMVEhZEAISEQoNSEVBTFRIX1BBVVNFRBADMuwGCg5DbHVzdGVyU2VydmljZRJQChBXYXRjaERlcGxveW1lbnRzEiAuY3RybC52MS5XYXRjaERlcGxveW1lbnRzUmVxdWVzdBoYLmN0cmwudjEuRGVwbG95bWVudFN0YXRlMAESSgoOV2F0Y2hTZW50aW5lbHMSHi5jdHJsLnYxLldhdGNoU2VudGluZWxzUmVxdWVzdBoWLmN0cmwudjEuU2VudGluZWxTdGF0ZTABEloKF0dldERlc2lyZWRTZW50aW5lbFN0YXRlEicuY3RybC52MS5HZXREZXNpcmVkU2VudGluZWxTdGF0ZVJlcXVlc3QaFi5jdHJsLnYxLlNlbnRpbmVsU3RhdGUSYwoUUmVwb3J0U2VudGluZWxTdGF0dXMSJC5jdHJsLnYxLlJlcG9ydFNlbnRpbmVsU3RhdHVzUmVxdWVzdBolLmN0cmwudjEuUmVwb3J0U2VudGluZWxTdGF0dXNSZXNwb25zZRJgChlHZXREZXNpcmVkRGVwbG95bWVudFN0YXRlEikuY3RybC52MS5HZXREZXNpcmVkRGVwbG95bWVudFN0YXRlUmVxdWVzdBoYLmN0cmwudjEuRGVwbG95bWVudFN0YXRlEmkKFlJlcG9ydERlcGxveW1lbnRTdGF0dXMSJi5jdHJsLnYxLlJlcG9ydERlcGxveW1lbnRTdGF0dXNSZXF1ZXN0GicuY3RybC52MS5SZXBvcnREZXBsb3ltZW50U3RhdHVzUmVzcG9uc2USbQoaV2F0Y2hDaWxpdW1OZXR3b3JrUG9saWNpZXMSKi5jdHJsLnYxLldhdGNoQ2lsaXVtTmV0d29ya1BvbGljaWVzUmVxdWVzdBohLmN0cmwudjEuQ2lsaXVtTmV0d29ya1BvbGljeVN0YXRlMAESewoiR2V0RGVzaXJlZENpbGl1bU5ldHdvcmtQb2xpY3lTdGF0ZRIyLmN0cmwudjEuR2V0RGVzaXJlZENpbGl1bU5ldHdvcmtQb2xpY3lTdGF0ZVJlcXVlc3QaIS5jdHJsLnYxLkNpbGl1bU5ldHdvcmtQb2xpY3lTdGF0ZRJCCglIZWFydGJlYXQSGS5jdHJsLnYxLkhlYXJ0YmVhdFJlcXVlc3QaGi5jdHJsLnYxLkhlYXJ0YmVhdFJlc3BvbnNlQosBCgtjb20uY3RybC52MUIMQ2x1c3RlclByb3RvUAFaMWdpdGh1Yi5jb20vdW5rZXllZC91bmtleS9nZW4vcHJvdG8vY3RybC92MTtjdHJsdjGiAgNDWFiqAgdDdHJsLlYxygIHQ3RybFxWMeICE0N0cmxcVjFcR1BCTWV0YWRhdGHqAghDdHJsOjpWMWIGcHJvdG8z"); /** * @generated from message ctrl.v1.WatchCiliumNetworkPoliciesRequest @@ -812,6 +812,49 @@ export type DeleteDeployment = Message<"ctrl.v1.DeleteDeployment"> & { export const DeleteDeploymentSchema: GenMessage = /*@__PURE__*/ messageDesc(file_ctrl_v1_cluster, 18); +/** + * HeartbeatRequest is sent periodically by krane agents to register their + * presence. The control plane uses this to populate cluster_regions and + * clusters tables. + * + * @generated from message ctrl.v1.HeartbeatRequest + */ +export type HeartbeatRequest = Message<"ctrl.v1.HeartbeatRequest"> & { + /** + * region is the geographic region name (e.g. "us-east-1"). + * + * @generated from field: string region = 1; + */ + region: string; + + /** + * platform is the infrastructure provider (e.g. "aws", "gcp", "local"). + * + * @generated from field: string platform = 2; + */ + platform: string; +}; + +/** + * Describes the message ctrl.v1.HeartbeatRequest. + * Use `create(HeartbeatRequestSchema)` to create a new message. + */ +export const HeartbeatRequestSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_ctrl_v1_cluster, 19); + +/** + * @generated from message ctrl.v1.HeartbeatResponse + */ +export type HeartbeatResponse = Message<"ctrl.v1.HeartbeatResponse"> & { +}; + +/** + * Describes the message ctrl.v1.HeartbeatResponse. + * Use `create(HeartbeatResponseSchema)` to create a new message. + */ +export const HeartbeatResponseSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_ctrl_v1_cluster, 20); + /** * Health represents the health state of a resource (sentinel, deployment instance, etc.) * @@ -949,6 +992,18 @@ export const ClusterService: GenService<{ input: typeof GetDesiredCiliumNetworkPolicyStateRequestSchema; output: typeof CiliumNetworkPolicyStateSchema; }, + /** + * Heartbeat is called periodically by krane agents to register their cluster + * and region with the control plane. This populates the cluster_regions and + * clusters tables, making regions dynamically discoverable. + * + * @generated from rpc ctrl.v1.ClusterService.Heartbeat + */ + heartbeat: { + methodKind: "unary"; + input: typeof HeartbeatRequestSchema; + output: typeof HeartbeatResponseSchema; + }, }> = /*@__PURE__*/ serviceDesc(file_ctrl_v1_cluster, 0); diff --git a/web/apps/dashboard/lib/collections/deploy/environment-settings.ts b/web/apps/dashboard/lib/collections/deploy/environment-settings.ts index c00a81ba378..ea2f348bb9c 100644 --- a/web/apps/dashboard/lib/collections/deploy/environment-settings.ts +++ b/web/apps/dashboard/lib/collections/deploy/environment-settings.ts @@ -80,7 +80,14 @@ export const environmentSettings = createCollection environmentId, }); - return [flattenSettingsResponse(environmentId, result.buildSettings, result.runtimeSettings, result.regionalSettings)]; + return [ + flattenSettingsResponse( + environmentId, + result.buildSettings, + result.runtimeSettings, + result.regionalSettings, + ), + ]; }, getKey: (item) => item.environmentId, id: "environmentSettings", @@ -216,7 +223,8 @@ export function buildSettingsMutations( const origReplicas = original.regions.at(0)?.replicas ?? 1; const modReplicas = modified.regions.at(0)?.replicas ?? 1; - const instancesChanged = !regionsChanged && modified.regions.length > 0 && modReplicas !== origReplicas; + const instancesChanged = + !regionsChanged && modified.regions.length > 0 && modReplicas !== origReplicas; if (instancesChanged) { mutations.push( diff --git a/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/runtime/update-regions.ts b/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/runtime/update-regions.ts index da55d8400f2..39ef1a0251c 100644 --- a/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/runtime/update-regions.ts +++ b/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/runtime/update-regions.ts @@ -1,7 +1,7 @@ -import { and, db, eq, inArray, notInArray } from "@/lib/db"; +import { and, db, eq, notInArray } from "@/lib/db"; +import { TRPCError } from "@trpc/server"; import { appRegionalSettings, environments } from "@unkey/db/src/schema"; import { z } from "zod"; -import { TRPCError } from "@trpc/server"; import { workspaceProcedure } from "../../../../trpc"; import { resolveProjectEnvironmentIds } from "../utils"; @@ -41,13 +41,15 @@ export const updateRegions = workspaceProcedure for (const envId of envIds) { // Delete rows for regions that are no longer selected - await db.delete(appRegionalSettings).where( - and( - eq(appRegionalSettings.workspaceId, ctx.workspace.id), - eq(appRegionalSettings.environmentId, envId), - notInArray(appRegionalSettings.regionId, input.regionIds), - ), - ); + await db + .delete(appRegionalSettings) + .where( + and( + eq(appRegionalSettings.workspaceId, ctx.workspace.id), + eq(appRegionalSettings.environmentId, envId), + notInArray(appRegionalSettings.regionId, input.regionIds), + ), + ); // Insert rows for newly added regions for (const regionId of input.regionIds) { diff --git a/web/internal/db/src/schema/app_regional_settings.ts b/web/internal/db/src/schema/app_regional_settings.ts index d93c73922e4..e7b12d38e19 100644 --- a/web/internal/db/src/schema/app_regional_settings.ts +++ b/web/internal/db/src/schema/app_regional_settings.ts @@ -1,12 +1,5 @@ import { relations } from "drizzle-orm"; -import { - bigint, - index, - int, - mysqlTable, - uniqueIndex, - varchar, -} from "drizzle-orm/mysql-core"; +import { bigint, index, int, mysqlTable, uniqueIndex, varchar } from "drizzle-orm/mysql-core"; import { apps } from "./apps"; import { environments } from "./environments"; import { lifecycleDates } from "./util/lifecycle_dates"; @@ -21,9 +14,7 @@ import { workspaces } from "./workspaces"; export const appRegionalSettings = mysqlTable( "app_regional_settings", { - pk: bigint("pk", { mode: "number", unsigned: true }) - .autoincrement() - .primaryKey(), + pk: bigint("pk", { mode: "number", unsigned: true }).autoincrement().primaryKey(), workspaceId: varchar("workspace_id", { length: 256 }).notNull(), appId: varchar("app_id", { length: 64 }).notNull(), @@ -38,29 +29,22 @@ export const appRegionalSettings = mysqlTable( ...lifecycleDates, }, (table) => [ - uniqueIndex("unique_app_env_region").on( - table.appId, - table.environmentId, - table.regionId, - ), + uniqueIndex("unique_app_env_region").on(table.appId, table.environmentId, table.regionId), index("workspace_idx").on(table.workspaceId), ], ); -export const appRegionalSettingsRelations = relations( - appRegionalSettings, - ({ one }) => ({ - workspace: one(workspaces, { - fields: [appRegionalSettings.workspaceId], - references: [workspaces.id], - }), - app: one(apps, { - fields: [appRegionalSettings.appId], - references: [apps.id], - }), - environment: one(environments, { - fields: [appRegionalSettings.environmentId], - references: [environments.id], - }), +export const appRegionalSettingsRelations = relations(appRegionalSettings, ({ one }) => ({ + workspace: one(workspaces, { + fields: [appRegionalSettings.workspaceId], + references: [workspaces.id], }), -); + app: one(apps, { + fields: [appRegionalSettings.appId], + references: [apps.id], + }), + environment: one(environments, { + fields: [appRegionalSettings.environmentId], + references: [environments.id], + }), +})); diff --git a/web/internal/db/src/schema/horizontal_autoscaling_policies.ts b/web/internal/db/src/schema/horizontal_autoscaling_policies.ts index ee52f90a87c..fdf413091ff 100644 --- a/web/internal/db/src/schema/horizontal_autoscaling_policies.ts +++ b/web/internal/db/src/schema/horizontal_autoscaling_policies.ts @@ -1,12 +1,5 @@ import { relations } from "drizzle-orm"; -import { - bigint, - index, - int, - mysqlTable, - tinyint, - varchar, -} from "drizzle-orm/mysql-core"; +import { bigint, index, int, mysqlTable, tinyint, varchar } from "drizzle-orm/mysql-core"; import { lifecycleDates } from "./util/lifecycle_dates"; import { workspaces } from "./workspaces"; @@ -34,14 +27,15 @@ export const horizontalAutoscalingPolicies = mysqlTable( ...lifecycleDates, }, - (table) => [ - index("workspace_idx").on(table.workspaceId), - ], + (table) => [index("workspace_idx").on(table.workspaceId)], ); -export const horizontalAutoscalingPoliciesRelations = relations(horizontalAutoscalingPolicies, ({ one }) => ({ - workspace: one(workspaces, { - fields: [horizontalAutoscalingPolicies.workspaceId], - references: [workspaces.id], +export const horizontalAutoscalingPoliciesRelations = relations( + horizontalAutoscalingPolicies, + ({ one }) => ({ + workspace: one(workspaces, { + fields: [horizontalAutoscalingPolicies.workspaceId], + references: [workspaces.id], + }), }), -})); +); From 837cab66ea67edde01ca7510e36eb23f28932123 Mon Sep 17 00:00:00 2001 From: chronark Date: Sat, 7 Mar 2026 13:15:03 +0100 Subject: [PATCH 14/24] feat: add heartbeat loop to krane agent Krane now sends periodic heartbeats to the control plane every 30s, reporting its region and platform. Also adds Platform config field, fixes duplicate r.Recover()/r.DeferCtx calls, and updates mock client. --- pkg/db/models_generated.go | 42 +++++++++++++++++++++++--------------- pkg/db/schema.sql | 31 +++++++++++++++++++--------- svc/krane/BUILD.bazel | 2 ++ svc/krane/config.go | 3 +++ svc/krane/run.go | 17 +++++++++++---- 5 files changed, 64 insertions(+), 31 deletions(-) diff --git a/pkg/db/models_generated.go b/pkg/db/models_generated.go index 15185513130..ea4ec21e1cf 100644 --- a/pkg/db/models_generated.go +++ b/pkg/db/models_generated.go @@ -1002,6 +1002,18 @@ type AppEnvironmentVariable struct { UpdatedAt sql.NullInt64 `db:"updated_at"` } +type AppRegionalSetting struct { + Pk uint64 `db:"pk"` + WorkspaceID string `db:"workspace_id"` + AppID string `db:"app_id"` + EnvironmentID string `db:"environment_id"` + RegionID string `db:"region_id"` + Replicas int32 `db:"replicas"` + HorizontalAutoscalingPolicyID sql.NullString `db:"horizontal_autoscaling_policy_id"` + CreatedAt int64 `db:"created_at"` + UpdatedAt sql.NullInt64 `db:"updated_at"` +} + type AppRuntimeSetting struct { Pk uint64 `db:"pk"` WorkspaceID string `db:"workspace_id"` @@ -1019,23 +1031,6 @@ type AppRuntimeSetting struct { UpdatedAt sql.NullInt64 `db:"updated_at"` } -type AppScalingSetting struct { - Pk uint64 `db:"pk"` - WorkspaceID string `db:"workspace_id"` - AppID string `db:"app_id"` - EnvironmentID string `db:"environment_id"` - RegionID string `db:"region_id"` - MemoryThreshold sql.NullInt16 `db:"memory_threshold"` - CpuThreshold sql.NullInt16 `db:"cpu_threshold"` - RpsThreshold sql.NullInt16 `db:"rps_threshold"` - ReplicasMin int32 `db:"replicas_min"` - ReplicasMax int32 `db:"replicas_max"` - MaxCpuMillicores int32 `db:"max_cpu_millicores"` - MaxMemoryMib int32 `db:"max_memory_mib"` - CreatedAt int64 `db:"created_at"` - UpdatedAt sql.NullInt64 `db:"updated_at"` -} - type AuditLog struct { Pk uint64 `db:"pk"` ID string `db:"id"` @@ -1271,6 +1266,19 @@ type GithubRepoConnection struct { UpdatedAt sql.NullInt64 `db:"updated_at"` } +type HorizontalAutoscalingPolicy struct { + Pk uint64 `db:"pk"` + ID string `db:"id"` + WorkspaceID string `db:"workspace_id"` + ReplicasMin int32 `db:"replicas_min"` + ReplicasMax int32 `db:"replicas_max"` + MemoryThreshold sql.NullInt16 `db:"memory_threshold"` + CpuThreshold sql.NullInt16 `db:"cpu_threshold"` + RpsThreshold sql.NullInt16 `db:"rps_threshold"` + CreatedAt int64 `db:"created_at"` + UpdatedAt sql.NullInt64 `db:"updated_at"` +} + type Identity struct { Pk uint64 `db:"pk"` ID string `db:"id"` diff --git a/pkg/db/schema.sql b/pkg/db/schema.sql index 916d6a6406c..73b1b71404c 100644 --- a/pkg/db/schema.sql +++ b/pkg/db/schema.sql @@ -448,22 +448,17 @@ CREATE TABLE `app_runtime_settings` ( CONSTRAINT `app_runtime_settings_app_env_idx` UNIQUE(`app_id`,`environment_id`) ); -CREATE TABLE `app_scaling_settings` ( +CREATE TABLE `app_regional_settings` ( `pk` bigint unsigned AUTO_INCREMENT NOT NULL, `workspace_id` varchar(256) NOT NULL, `app_id` varchar(64) NOT NULL, `environment_id` varchar(128) NOT NULL, `region_id` varchar(64) NOT NULL, - `memory_threshold` tinyint, - `cpu_threshold` tinyint, - `rps_threshold` tinyint, - `replicas_min` int NOT NULL, - `replicas_max` int NOT NULL, - `max_cpu_millicores` int NOT NULL, - `max_memory_mib` int NOT NULL, + `replicas` int NOT NULL DEFAULT 1, + `horizontal_autoscaling_policy_id` varchar(64), `created_at` bigint NOT NULL, `updated_at` bigint, - CONSTRAINT `app_scaling_settings_pk` PRIMARY KEY(`pk`), + CONSTRAINT `app_regional_settings_pk` PRIMARY KEY(`pk`), CONSTRAINT `unique_app_env_region` UNIQUE(`app_id`,`environment_id`,`region_id`) ); @@ -743,6 +738,21 @@ CREATE TABLE `cluster_regions` ( CONSTRAINT `unique_reqion_per_platform` UNIQUE(`name`,`platform`) ); +CREATE TABLE `horizontal_autoscaling_policies` ( + `pk` bigint unsigned AUTO_INCREMENT NOT NULL, + `id` varchar(64) NOT NULL, + `workspace_id` varchar(256) NOT NULL, + `replicas_min` int NOT NULL, + `replicas_max` int NOT NULL, + `memory_threshold` tinyint, + `cpu_threshold` tinyint, + `rps_threshold` tinyint, + `created_at` bigint NOT NULL, + `updated_at` bigint, + CONSTRAINT `horizontal_autoscaling_policies_pk` PRIMARY KEY(`pk`), + CONSTRAINT `horizontal_autoscaling_policies_id_unique` UNIQUE(`id`) +); + CREATE INDEX `workspace_id_idx` ON `apis` (`workspace_id`); CREATE INDEX `workspace_id_idx` ON `roles` (`workspace_id`); CREATE INDEX `key_auth_id_deleted_at_idx` ON `keys` (`key_auth_id`,`deleted_at_m`); @@ -762,7 +772,7 @@ CREATE INDEX `bucket` ON `audit_log_target` (`bucket`); CREATE INDEX `id_idx` ON `audit_log_target` (`id`); CREATE INDEX `environments_project_idx` ON `environments` (`project_id`); CREATE INDEX `apps_workspace_idx` ON `apps` (`workspace_id`); -CREATE INDEX `workspace_idx` ON `app_scaling_settings` (`workspace_id`); +CREATE INDEX `workspace_idx` ON `app_regional_settings` (`workspace_id`); CREATE INDEX `workspace_idx` ON `deployments` (`workspace_id`); CREATE INDEX `project_idx` ON `deployments` (`project_id`); CREATE INDEX `status_idx` ON `deployments` (`status`); @@ -781,4 +791,5 @@ CREATE INDEX `idx_region` ON `instances` (`region`); CREATE INDEX `environment_id_idx` ON `frontline_routes` (`environment_id`); CREATE INDEX `deployment_id_idx` ON `frontline_routes` (`deployment_id`); CREATE INDEX `installation_id_idx` ON `github_repo_connections` (`installation_id`); +CREATE INDEX `workspace_idx` ON `horizontal_autoscaling_policies` (`workspace_id`); diff --git a/svc/krane/BUILD.bazel b/svc/krane/BUILD.bazel index 8b197c4f376..8b100c4d494 100644 --- a/svc/krane/BUILD.bazel +++ b/svc/krane/BUILD.bazel @@ -10,6 +10,7 @@ go_library( importpath = "github.com/unkeyed/unkey/svc/krane", visibility = ["//visibility:public"], deps = [ + "//gen/proto/ctrl/v1:ctrl", "//gen/proto/krane/v1/kranev1connect", "//gen/proto/vault/v1/vaultv1connect", "//gen/rpc/vault", @@ -17,6 +18,7 @@ go_library( "//pkg/config", "//pkg/logger", "//pkg/otel", + "//pkg/repeat", "//pkg/prometheus", "//pkg/rpc/interceptor", "//pkg/runner", diff --git a/svc/krane/config.go b/svc/krane/config.go index ec3b20c01f9..fb4a86e0828 100644 --- a/svc/krane/config.go +++ b/svc/krane/config.go @@ -37,6 +37,9 @@ type Config struct { // Region identifies the geographic region where this node is deployed. Region string `toml:"region" config:"required,nonempty"` + // Platform identifies the infrastructure provider (e.g. "aws", "gcp", "local"). + Platform string `toml:"platform" config:"required,nonempty"` + // RPCPort is the TCP port for the gRPC server. RPCPort int `toml:"rpc_port" config:"default=8070,min=1,max=65535"` diff --git a/svc/krane/run.go b/svc/krane/run.go index b5fdefd7dff..e9124e3968d 100644 --- a/svc/krane/run.go +++ b/svc/krane/run.go @@ -10,10 +10,12 @@ import ( "time" "connectrpc.com/connect" + ctrlv1 "github.com/unkeyed/unkey/gen/proto/ctrl/v1" "github.com/unkeyed/unkey/gen/proto/krane/v1/kranev1connect" "github.com/unkeyed/unkey/gen/proto/vault/v1/vaultv1connect" "github.com/unkeyed/unkey/gen/rpc/vault" "github.com/unkeyed/unkey/pkg/logger" + "github.com/unkeyed/unkey/pkg/repeat" "github.com/unkeyed/unkey/pkg/otel" "github.com/unkeyed/unkey/pkg/prometheus" "github.com/unkeyed/unkey/pkg/rpc/interceptor" @@ -90,10 +92,6 @@ func Run(ctx context.Context, cfg Config) error { r.DeferCtx(shutdownGrafana) - defer r.Recover() - - r.DeferCtx(shutdownGrafana) - cluster := controlplane.NewClient(controlplane.ClientConfig{ URL: cfg.Control.URL, BearerToken: cfg.Control.Token, @@ -151,6 +149,17 @@ func Run(ctx context.Context, cfg Config) error { } r.Defer(sentinelCtrl.Stop) + // Start heartbeat loop to register this cluster with the control plane + stopHeartbeat := repeat.Every(30*time.Second, func() { + if _, err := cluster.Heartbeat(ctx, &ctrlv1.HeartbeatRequest{ + Region: cfg.Region, + Platform: cfg.Platform, + }); err != nil { + logger.Warn("heartbeat failed", "error", err) + } + }) + r.Defer(func() error { stopHeartbeat(); return nil }) + // Create vault client for secrets decryption var vaultClient vault.VaultServiceClient if cfg.Vault.URL != "" { From 6c5eabc0d4873659fe2bdd92c7b39674d7c42892 Mon Sep 17 00:00:00 2001 From: chronark Date: Sat, 7 Mar 2026 17:54:14 +0100 Subject: [PATCH 15/24] feat: implement Heartbeat RPC handler and rename cluster_regions to regions - Add Heartbeat handler on ctrl that upserts regions and clusters tables - Rename cluster_regions table to regions for clarity - Remove AvailableRegions config from ctrl api/worker - Deploy workflow now fails with terminal error if no regions configured - Certificate bootstrap queries regions from DB instead of config - Add SQL queries: UpsertRegion, FindRegionByNameAndPlatform, ListRegions, UpsertCluster --- dev/config/ctrl-api.toml | 1 - dev/k8s/manifests/ctrl-api.yaml | 1 - dev/k8s/manifests/ctrl-worker.yaml | 1 - .../control-plane/api/configuration.mdx | 5 +- .../control-plane/worker/configuration.mdx | 2 - gen/proto/ctrl/v1/cluster.pb.go | 2 +- .../ctrl/v1/ctrlv1connect/cluster.connect.go | 4 +- pkg/db/BUILD.bazel | 6 ++ ...ulk_cluster_region_upsert.sql_generated.go | 40 +++++++++++ pkg/db/bulk_cluster_upsert.sql_generated.go | 46 +++++++++++++ ...uster_region_find_by_name.sql_generated.go | 29 ++++++++ pkg/db/cluster_region_list.sql_generated.go | 46 +++++++++++++ pkg/db/cluster_region_upsert.sql_generated.go | 48 +++++++++++++ pkg/db/cluster_upsert.sql_generated.go | 55 +++++++++++++++ pkg/db/models_generated.go | 14 ++-- pkg/db/querier_bulk_generated.go | 2 + pkg/db/querier_generated.go | 37 ++++++++++ .../queries/cluster_region_find_by_name.sql | 2 + pkg/db/queries/cluster_region_list.sql | 2 + pkg/db/queries/cluster_region_upsert.sql | 13 ++++ pkg/db/queries/cluster_upsert.sql | 14 ++++ pkg/db/schema.sql | 13 ++-- pkg/uid/prefix.go | 4 +- svc/ctrl/api/certificate.go | 12 ++-- svc/ctrl/api/config.go | 4 -- svc/ctrl/api/harness_test.go | 16 ++--- svc/ctrl/api/run.go | 1 - svc/ctrl/integration/harness/harness.go | 12 ++-- svc/ctrl/proto/ctrl/v1/cluster.proto | 4 +- svc/ctrl/services/cluster/BUILD.bazel | 1 + svc/ctrl/services/cluster/rpc_heartbeat.go | 68 +++++++++++++++++++ svc/ctrl/services/deployment/doc.go | 2 +- svc/ctrl/worker/config.go | 4 -- svc/ctrl/worker/deploy/deploy_handler.go | 17 +++-- svc/ctrl/worker/deploy/service.go | 27 ++++---- svc/ctrl/worker/run.go | 10 +-- svc/krane/BUILD.bazel | 2 +- .../internal/testutil/mock_cluster_client.go | 2 +- svc/krane/run.go | 2 +- .../dashboard/gen/proto/ctrl/v1/cluster_pb.ts | 4 +- .../get-available-regions.ts | 4 +- web/internal/db/src/schema/clusters.ts | 8 +-- web/internal/db/src/schema/index.ts | 2 +- .../schema/{cluster_regions.ts => regions.ts} | 8 +-- 44 files changed, 498 insertions(+), 99 deletions(-) create mode 100644 pkg/db/bulk_cluster_region_upsert.sql_generated.go create mode 100644 pkg/db/bulk_cluster_upsert.sql_generated.go create mode 100644 pkg/db/cluster_region_find_by_name.sql_generated.go create mode 100644 pkg/db/cluster_region_list.sql_generated.go create mode 100644 pkg/db/cluster_region_upsert.sql_generated.go create mode 100644 pkg/db/cluster_upsert.sql_generated.go create mode 100644 pkg/db/queries/cluster_region_find_by_name.sql create mode 100644 pkg/db/queries/cluster_region_list.sql create mode 100644 pkg/db/queries/cluster_region_upsert.sql create mode 100644 pkg/db/queries/cluster_upsert.sql create mode 100644 svc/ctrl/services/cluster/rpc_heartbeat.go rename web/internal/db/src/schema/{cluster_regions.ts => regions.ts} (73%) diff --git a/dev/config/ctrl-api.toml b/dev/config/ctrl-api.toml index fbb716d0f84..dfda5873950 100644 --- a/dev/config/ctrl-api.toml +++ b/dev/config/ctrl-api.toml @@ -2,7 +2,6 @@ instance_id = "ctrl-api-dev" region = "local" http_port = 7091 auth_token = "your-local-dev-key" -available_regions = ["local.dev"] default_domain = "unkey.local" cname_domain = "unkey.local" diff --git a/dev/k8s/manifests/ctrl-api.yaml b/dev/k8s/manifests/ctrl-api.yaml index 3f90aa3d386..22d3b502c71 100644 --- a/dev/k8s/manifests/ctrl-api.yaml +++ b/dev/k8s/manifests/ctrl-api.yaml @@ -12,7 +12,6 @@ data: region = "local" http_port = 7091 auth_token = "your-local-dev-key" - available_regions = ["local.dev"] default_domain = "unkey.local" cname_domain = "unkey.local" diff --git a/dev/k8s/manifests/ctrl-worker.yaml b/dev/k8s/manifests/ctrl-worker.yaml index d2fd62fcf72..bdefe0c050e 100644 --- a/dev/k8s/manifests/ctrl-worker.yaml +++ b/dev/k8s/manifests/ctrl-worker.yaml @@ -14,7 +14,6 @@ data: cname_domain = "unkey.local" build_platform = "linux/arm64" sentinel_image = "unkey/sentinel:latest" - available_regions = ["local.dev"] [database] primary = "unkey:password@tcp(mysql:3306)/unkey?parseTime=true&interpolateParams=true" diff --git a/docs/engineering/architecture/services/control-plane/api/configuration.mdx b/docs/engineering/architecture/services/control-plane/api/configuration.mdx index fc33416317e..398025cbf52 100644 --- a/docs/engineering/architecture/services/control-plane/api/configuration.mdx +++ b/docs/engineering/architecture/services/control-plane/api/configuration.mdx @@ -40,9 +40,7 @@ Rotation is manual today. There is no built-in rotation mechanism. TODO: Replace with JWT-based auth once `auth.unkey.cloud` is in place. - - Regions available for deployments. - + Base domain for wildcard certificates. @@ -117,7 +115,6 @@ prometheus_port = 9090 region = "${UNKEY_REGION}" instance_id = "${POD_NAME}" auth_token = "${UNKEY_AUTH_TOKEN}" -available_regions = ["eu-central-1.aws", "us-east-1.aws", "us-west-2.aws"] default_domain = "${UNKEY_DEFAULT_DOMAIN}" regional_domain = "${UNKEY_REGIONAL_DOMAIN}" cname_domain = "${UNKEY_CNAME_DOMAIN}" diff --git a/docs/engineering/architecture/services/control-plane/worker/configuration.mdx b/docs/engineering/architecture/services/control-plane/worker/configuration.mdx index c972d5572a0..76e47572999 100644 --- a/docs/engineering/architecture/services/control-plane/worker/configuration.mdx +++ b/docs/engineering/architecture/services/control-plane/worker/configuration.mdx @@ -38,7 +38,6 @@ These fields must be set for production deployments. | `default_domain` | string | `unkey.app` | Used for sentinel bootstrapping. | `build_platform` | string | `linux/amd64` | Build platform, format `linux/{arch}`. | `sentinel_image` | string | `ghcr.io/unkeyed/unkey:local` | Sentinel image override. -| `available_regions` | list | - | Regions allowed for deployments. | `acme` | object | - | ACME config for cert issuance. | `depot` | object | - | Depot.dev config for builds. | `registry` | object | - | Registry credentials for builds. @@ -126,7 +125,6 @@ instance_id = "${POD_NAME}" default_domain = "${UNKEY_DEFAULT_DOMAIN}" build_platform = "linux/amd64" sentinel_image = "ghcr.io/unkeyed/unkey:v2.0.77" -available_regions = ["eu-central-1.aws", "us-east-1.aws"] cname_domain = "${UNKEY_CNAME_DOMAIN}" [database] diff --git a/gen/proto/ctrl/v1/cluster.pb.go b/gen/proto/ctrl/v1/cluster.pb.go index 021d77df9ec..b11fb6b6e0c 100644 --- a/gen/proto/ctrl/v1/cluster.pb.go +++ b/gen/proto/ctrl/v1/cluster.pb.go @@ -1500,7 +1500,7 @@ func (x *DeleteDeployment) GetK8SName() string { } // HeartbeatRequest is sent periodically by krane agents to register their -// presence. The control plane uses this to populate cluster_regions and +// presence. The control plane uses this to populate regions and // clusters tables. type HeartbeatRequest struct { state protoimpl.MessageState `protogen:"open.v1"` diff --git a/gen/proto/ctrl/v1/ctrlv1connect/cluster.connect.go b/gen/proto/ctrl/v1/ctrlv1connect/cluster.connect.go index 5a1e137023f..955e733fed1 100644 --- a/gen/proto/ctrl/v1/ctrlv1connect/cluster.connect.go +++ b/gen/proto/ctrl/v1/ctrlv1connect/cluster.connect.go @@ -100,7 +100,7 @@ type ClusterServiceClient interface { // Used by the resync loop to verify consistency for existing resources. GetDesiredCiliumNetworkPolicyState(context.Context, *connect.Request[v1.GetDesiredCiliumNetworkPolicyStateRequest]) (*connect.Response[v1.CiliumNetworkPolicyState], error) // Heartbeat is called periodically by krane agents to register their cluster - // and region with the control plane. This populates the cluster_regions and + // and region with the control plane. This populates the regions and // clusters tables, making regions dynamically discoverable. Heartbeat(context.Context, *connect.Request[v1.HeartbeatRequest]) (*connect.Response[v1.HeartbeatResponse], error) } @@ -261,7 +261,7 @@ type ClusterServiceHandler interface { // Used by the resync loop to verify consistency for existing resources. GetDesiredCiliumNetworkPolicyState(context.Context, *connect.Request[v1.GetDesiredCiliumNetworkPolicyStateRequest]) (*connect.Response[v1.CiliumNetworkPolicyState], error) // Heartbeat is called periodically by krane agents to register their cluster - // and region with the control plane. This populates the cluster_regions and + // and region with the control plane. This populates the regions and // clusters tables, making regions dynamically discoverable. Heartbeat(context.Context, *connect.Request[v1.HeartbeatRequest]) (*connect.Response[v1.HeartbeatResponse], error) } diff --git a/pkg/db/BUILD.bazel b/pkg/db/BUILD.bazel index 11560d8e127..49fcb059158 100644 --- a/pkg/db/BUILD.bazel +++ b/pkg/db/BUILD.bazel @@ -51,6 +51,8 @@ go_library( "bulk_certificate_insert.sql_generated.go", "bulk_cilium_network_policy_insert.sql_generated.go", "bulk_clickhouse_workspace_settings_insert.sql_generated.go", + "bulk_cluster_region_upsert.sql_generated.go", + "bulk_cluster_upsert.sql_generated.go", "bulk_custom_domain_insert.sql_generated.go", "bulk_custom_domain_upsert.sql_generated.go", "bulk_deployment_insert.sql_generated.go", @@ -96,6 +98,10 @@ go_library( "clickhouse_workspace_settings_find_by_workspace_id.sql_generated.go", "clickhouse_workspace_settings_insert.sql_generated.go", "clickhouse_workspace_settings_update_limits.sql_generated.go", + "cluster_region_find_by_name.sql_generated.go", + "cluster_region_list.sql_generated.go", + "cluster_region_upsert.sql_generated.go", + "cluster_upsert.sql_generated.go", "custom_domain_delete_by_id.sql_generated.go", "custom_domain_find_by_domain.sql_generated.go", "custom_domain_find_by_domain_or_wildcard.sql_generated.go", diff --git a/pkg/db/bulk_cluster_region_upsert.sql_generated.go b/pkg/db/bulk_cluster_region_upsert.sql_generated.go new file mode 100644 index 00000000000..57e53a31744 --- /dev/null +++ b/pkg/db/bulk_cluster_region_upsert.sql_generated.go @@ -0,0 +1,40 @@ +// Code generated by sqlc bulk insert plugin. DO NOT EDIT. + +package db + +import ( + "context" + "fmt" + "strings" +) + +// bulkUpsertRegion is the base query for bulk insert +const bulkUpsertRegion = `INSERT INTO regions ( id, name, platform ) VALUES %s ON DUPLICATE KEY UPDATE name = name` + +// UpsertRegion performs bulk insert in a single query +func (q *BulkQueries) UpsertRegion(ctx context.Context, db DBTX, args []UpsertRegionParams) error { + + if len(args) == 0 { + return nil + } + + // Build the bulk insert query + valueClauses := make([]string, len(args)) + for i := range args { + valueClauses[i] = "( ?, ?, ? )" + } + + bulkQuery := fmt.Sprintf(bulkUpsertRegion, strings.Join(valueClauses, ", ")) + + // Collect all arguments + var allArgs []any + for _, arg := range args { + allArgs = append(allArgs, arg.ID) + allArgs = append(allArgs, arg.Name) + allArgs = append(allArgs, arg.Platform) + } + + // Execute the bulk insert + _, err := db.ExecContext(ctx, bulkQuery, allArgs...) + return err +} diff --git a/pkg/db/bulk_cluster_upsert.sql_generated.go b/pkg/db/bulk_cluster_upsert.sql_generated.go new file mode 100644 index 00000000000..a58cf2ee13b --- /dev/null +++ b/pkg/db/bulk_cluster_upsert.sql_generated.go @@ -0,0 +1,46 @@ +// Code generated by sqlc bulk insert plugin. DO NOT EDIT. + +package db + +import ( + "context" + "fmt" + "strings" +) + +// bulkUpsertCluster is the base query for bulk insert +const bulkUpsertCluster = `INSERT INTO clusters ( id, region_id, last_heartbeat_at ) VALUES %s ON DUPLICATE KEY UPDATE + last_heartbeat_at = ?` + +// UpsertCluster performs bulk insert in a single query +func (q *BulkQueries) UpsertCluster(ctx context.Context, db DBTX, args []UpsertClusterParams) error { + + if len(args) == 0 { + return nil + } + + // Build the bulk insert query + valueClauses := make([]string, len(args)) + for i := range args { + valueClauses[i] = "( ?, ?, ? )" + } + + bulkQuery := fmt.Sprintf(bulkUpsertCluster, strings.Join(valueClauses, ", ")) + + // Collect all arguments + var allArgs []any + for _, arg := range args { + allArgs = append(allArgs, arg.ID) + allArgs = append(allArgs, arg.RegionID) + allArgs = append(allArgs, arg.LastHeartbeatAt) + } + + // Add ON DUPLICATE KEY UPDATE parameters (only once, not per row) + if len(args) > 0 { + allArgs = append(allArgs, args[0].LastHeartbeatAt) + } + + // Execute the bulk insert + _, err := db.ExecContext(ctx, bulkQuery, allArgs...) + return err +} diff --git a/pkg/db/cluster_region_find_by_name.sql_generated.go b/pkg/db/cluster_region_find_by_name.sql_generated.go new file mode 100644 index 00000000000..216404ba66f --- /dev/null +++ b/pkg/db/cluster_region_find_by_name.sql_generated.go @@ -0,0 +1,29 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.30.0 +// source: cluster_region_find_by_name.sql + +package db + +import ( + "context" +) + +const findRegionByNameAndPlatform = `-- name: FindRegionByNameAndPlatform :one +SELECT id FROM regions WHERE name = ? AND platform = ? +` + +type FindRegionByNameAndPlatformParams struct { + Name string `db:"name"` + Platform string `db:"platform"` +} + +// FindRegionByNameAndPlatform +// +// SELECT id FROM regions WHERE name = ? AND platform = ? +func (q *Queries) FindRegionByNameAndPlatform(ctx context.Context, db DBTX, arg FindRegionByNameAndPlatformParams) (string, error) { + row := db.QueryRowContext(ctx, findRegionByNameAndPlatform, arg.Name, arg.Platform) + var id string + err := row.Scan(&id) + return id, err +} diff --git a/pkg/db/cluster_region_list.sql_generated.go b/pkg/db/cluster_region_list.sql_generated.go new file mode 100644 index 00000000000..dbbed0edea8 --- /dev/null +++ b/pkg/db/cluster_region_list.sql_generated.go @@ -0,0 +1,46 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.30.0 +// source: cluster_region_list.sql + +package db + +import ( + "context" +) + +const listRegions = `-- name: ListRegions :many +SELECT id, name, platform FROM regions +` + +type ListRegionsRow struct { + ID string `db:"id"` + Name string `db:"name"` + Platform string `db:"platform"` +} + +// ListRegions +// +// SELECT id, name, platform FROM regions +func (q *Queries) ListRegions(ctx context.Context, db DBTX) ([]ListRegionsRow, error) { + rows, err := db.QueryContext(ctx, listRegions) + if err != nil { + return nil, err + } + defer rows.Close() + var items []ListRegionsRow + for rows.Next() { + var i ListRegionsRow + if err := rows.Scan(&i.ID, &i.Name, &i.Platform); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} diff --git a/pkg/db/cluster_region_upsert.sql_generated.go b/pkg/db/cluster_region_upsert.sql_generated.go new file mode 100644 index 00000000000..d70020b0662 --- /dev/null +++ b/pkg/db/cluster_region_upsert.sql_generated.go @@ -0,0 +1,48 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.30.0 +// source: cluster_region_upsert.sql + +package db + +import ( + "context" +) + +const upsertRegion = `-- name: UpsertRegion :exec +INSERT INTO regions ( + id, + name, + platform +) +VALUES ( + ?, + ?, + ? +) +ON DUPLICATE KEY UPDATE name = name +` + +type UpsertRegionParams struct { + ID string `db:"id"` + Name string `db:"name"` + Platform string `db:"platform"` +} + +// Inserts a region or does nothing if it already exists. +// +// INSERT INTO regions ( +// id, +// name, +// platform +// ) +// VALUES ( +// ?, +// ?, +// ? +// ) +// ON DUPLICATE KEY UPDATE name = name +func (q *Queries) UpsertRegion(ctx context.Context, db DBTX, arg UpsertRegionParams) error { + _, err := db.ExecContext(ctx, upsertRegion, arg.ID, arg.Name, arg.Platform) + return err +} diff --git a/pkg/db/cluster_upsert.sql_generated.go b/pkg/db/cluster_upsert.sql_generated.go new file mode 100644 index 00000000000..fba77920795 --- /dev/null +++ b/pkg/db/cluster_upsert.sql_generated.go @@ -0,0 +1,55 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.30.0 +// source: cluster_upsert.sql + +package db + +import ( + "context" +) + +const upsertCluster = `-- name: UpsertCluster :exec +INSERT INTO clusters ( + id, + region_id, + last_heartbeat_at +) +VALUES ( + ?, + ?, + ? +) +ON DUPLICATE KEY UPDATE + last_heartbeat_at = ? +` + +type UpsertClusterParams struct { + ID string `db:"id"` + RegionID string `db:"region_id"` + LastHeartbeatAt uint64 `db:"last_heartbeat_at"` +} + +// Upserts a cluster by region_id. If the cluster already exists, updates the heartbeat timestamp. +// +// INSERT INTO clusters ( +// id, +// region_id, +// last_heartbeat_at +// ) +// VALUES ( +// ?, +// ?, +// ? +// ) +// ON DUPLICATE KEY UPDATE +// last_heartbeat_at = ? +func (q *Queries) UpsertCluster(ctx context.Context, db DBTX, arg UpsertClusterParams) error { + _, err := db.ExecContext(ctx, upsertCluster, + arg.ID, + arg.RegionID, + arg.LastHeartbeatAt, + arg.LastHeartbeatAt, + ) + return err +} diff --git a/pkg/db/models_generated.go b/pkg/db/models_generated.go index ea4ec21e1cf..87ca16e47de 100644 --- a/pkg/db/models_generated.go +++ b/pkg/db/models_generated.go @@ -1126,13 +1126,6 @@ type Cluster struct { LastHeartbeatAt uint64 `db:"last_heartbeat_at"` } -type ClusterRegion struct { - Pk uint64 `db:"pk"` - ID string `db:"id"` - Name string `db:"name"` - Platform string `db:"platform"` -} - type CustomDomain struct { Pk uint64 `db:"pk"` ID string `db:"id"` @@ -1455,6 +1448,13 @@ type RatelimitOverride struct { DeletedAtM sql.NullInt64 `db:"deleted_at_m"` } +type Region struct { + Pk uint64 `db:"pk"` + ID string `db:"id"` + Name string `db:"name"` + Platform string `db:"platform"` +} + type Role struct { Pk uint64 `db:"pk"` ID string `db:"id"` diff --git a/pkg/db/querier_bulk_generated.go b/pkg/db/querier_bulk_generated.go index 92409f4b208..f2055074c90 100644 --- a/pkg/db/querier_bulk_generated.go +++ b/pkg/db/querier_bulk_generated.go @@ -17,6 +17,8 @@ type BulkQuerier interface { InsertCertificates(ctx context.Context, db DBTX, args []InsertCertificateParams) error InsertCiliumNetworkPolicies(ctx context.Context, db DBTX, args []InsertCiliumNetworkPolicyParams) error InsertClickhouseWorkspaceSettingses(ctx context.Context, db DBTX, args []InsertClickhouseWorkspaceSettingsParams) error + UpsertRegion(ctx context.Context, db DBTX, args []UpsertRegionParams) error + UpsertCluster(ctx context.Context, db DBTX, args []UpsertClusterParams) error InsertCustomDomains(ctx context.Context, db DBTX, args []InsertCustomDomainParams) error UpsertCustomDomain(ctx context.Context, db DBTX, args []UpsertCustomDomainParams) error InsertDeployments(ctx context.Context, db DBTX, args []InsertDeploymentParams) error diff --git a/pkg/db/querier_generated.go b/pkg/db/querier_generated.go index bd65dab2391..b9209de06b6 100644 --- a/pkg/db/querier_generated.go +++ b/pkg/db/querier_generated.go @@ -1020,6 +1020,10 @@ type Querier interface { // AND namespace_id = ? // AND identifier = ? FindRatelimitOverrideByIdentifier(ctx context.Context, db DBTX, arg FindRatelimitOverrideByIdentifierParams) (RatelimitOverride, error) + //FindRegionByNameAndPlatform + // + // SELECT id FROM regions WHERE name = ? AND platform = ? + FindRegionByNameAndPlatform(ctx context.Context, db DBTX, arg FindRegionByNameAndPlatformParams) (string, error) // Finds a role record by its ID // Returns: The role record if found // @@ -2267,6 +2271,10 @@ type Querier interface { // WHERE key_id IN (/*SLICE:key_ids*/?) // ORDER BY key_id, id ListRatelimitsByKeyIDs(ctx context.Context, db DBTX, keyIds []sql.NullString) ([]ListRatelimitsByKeyIDsRow, error) + //ListRegions + // + // SELECT id, name, platform FROM regions + ListRegions(ctx context.Context, db DBTX) ([]ListRegionsRow, error) //ListRepoConnectionDeployContexts // // SELECT @@ -2824,6 +2832,21 @@ type Querier interface { // sentinel_config = VALUES(sentinel_config), // updated_at = VALUES(updated_at) UpsertAppRuntimeSettings(ctx context.Context, db DBTX, arg UpsertAppRuntimeSettingsParams) error + // Upserts a cluster by region_id. If the cluster already exists, updates the heartbeat timestamp. + // + // INSERT INTO clusters ( + // id, + // region_id, + // last_heartbeat_at + // ) + // VALUES ( + // ?, + // ?, + // ? + // ) + // ON DUPLICATE KEY UPDATE + // last_heartbeat_at = ? + UpsertCluster(ctx context.Context, db DBTX, arg UpsertClusterParams) error //UpsertCustomDomain // // INSERT INTO custom_domains ( @@ -2939,6 +2962,20 @@ type Querier interface { // ratelimit_api_limit = VALUES(ratelimit_api_limit), // ratelimit_api_duration = VALUES(ratelimit_api_duration) UpsertQuota(ctx context.Context, db DBTX, arg UpsertQuotaParams) error + // Inserts a region or does nothing if it already exists. + // + // INSERT INTO regions ( + // id, + // name, + // platform + // ) + // VALUES ( + // ?, + // ?, + // ? + // ) + // ON DUPLICATE KEY UPDATE name = name + UpsertRegion(ctx context.Context, db DBTX, arg UpsertRegionParams) error //UpsertWorkspace // // INSERT INTO workspaces ( diff --git a/pkg/db/queries/cluster_region_find_by_name.sql b/pkg/db/queries/cluster_region_find_by_name.sql new file mode 100644 index 00000000000..88423b9f607 --- /dev/null +++ b/pkg/db/queries/cluster_region_find_by_name.sql @@ -0,0 +1,2 @@ +-- name: FindRegionByNameAndPlatform :one +SELECT id FROM regions WHERE name = sqlc.arg(name) AND platform = sqlc.arg(platform); diff --git a/pkg/db/queries/cluster_region_list.sql b/pkg/db/queries/cluster_region_list.sql new file mode 100644 index 00000000000..58d7b226378 --- /dev/null +++ b/pkg/db/queries/cluster_region_list.sql @@ -0,0 +1,2 @@ +-- name: ListRegions :many +SELECT id, name, platform FROM regions; diff --git a/pkg/db/queries/cluster_region_upsert.sql b/pkg/db/queries/cluster_region_upsert.sql new file mode 100644 index 00000000000..b7c436fb259 --- /dev/null +++ b/pkg/db/queries/cluster_region_upsert.sql @@ -0,0 +1,13 @@ +-- name: UpsertRegion :exec +-- Inserts a region or does nothing if it already exists. +INSERT INTO regions ( + id, + name, + platform +) +VALUES ( + sqlc.arg(id), + sqlc.arg(name), + sqlc.arg(platform) +) +ON DUPLICATE KEY UPDATE name = name; diff --git a/pkg/db/queries/cluster_upsert.sql b/pkg/db/queries/cluster_upsert.sql new file mode 100644 index 00000000000..51a365f63a9 --- /dev/null +++ b/pkg/db/queries/cluster_upsert.sql @@ -0,0 +1,14 @@ +-- name: UpsertCluster :exec +-- Upserts a cluster by region_id. If the cluster already exists, updates the heartbeat timestamp. +INSERT INTO clusters ( + id, + region_id, + last_heartbeat_at +) +VALUES ( + sqlc.arg(id), + sqlc.arg(region_id), + sqlc.arg(last_heartbeat_at) +) +ON DUPLICATE KEY UPDATE + last_heartbeat_at = sqlc.arg(last_heartbeat_at); diff --git a/pkg/db/schema.sql b/pkg/db/schema.sql index 73b1b71404c..ce8179283f5 100644 --- a/pkg/db/schema.sql +++ b/pkg/db/schema.sql @@ -724,18 +724,19 @@ CREATE TABLE `clusters` ( `region_id` varchar(64) NOT NULL, `last_heartbeat_at` bigint unsigned NOT NULL, CONSTRAINT `clusters_pk` PRIMARY KEY(`pk`), - CONSTRAINT `clusters_id_unique` UNIQUE(`id`) + CONSTRAINT `clusters_id_unique` UNIQUE(`id`), + CONSTRAINT `clusters_region_id_unique` UNIQUE(`region_id`) ); -CREATE TABLE `cluster_regions` ( +CREATE TABLE `regions` ( `pk` bigint unsigned AUTO_INCREMENT NOT NULL, `id` varchar(64) NOT NULL, `name` varchar(64) NOT NULL, `platform` varchar(64) NOT NULL, - CONSTRAINT `cluster_regions_pk` PRIMARY KEY(`pk`), - CONSTRAINT `cluster_regions_id_unique` UNIQUE(`id`), - CONSTRAINT `cluster_regions_name_unique` UNIQUE(`name`), - CONSTRAINT `unique_reqion_per_platform` UNIQUE(`name`,`platform`) + CONSTRAINT `regions_pk` PRIMARY KEY(`pk`), + CONSTRAINT `regions_id_unique` UNIQUE(`id`), + CONSTRAINT `regions_name_unique` UNIQUE(`name`), + CONSTRAINT `unique_region_per_platform` UNIQUE(`name`,`platform`) ); CREATE TABLE `horizontal_autoscaling_policies` ( diff --git a/pkg/uid/prefix.go b/pkg/uid/prefix.go index 942b7ceb422..4edb4dcf685 100644 --- a/pkg/uid/prefix.go +++ b/pkg/uid/prefix.go @@ -18,8 +18,10 @@ const ( RatelimitPrefix Prefix = "rl" AuditLogPrefix Prefix = "log" InstancePrefix Prefix = "ins" - SentinelPrefix Prefix = "se" + SentinelPrefix Prefix = "s" CiliumNetworkPolicyPrefix Prefix = "net" + ClusterPrefix Prefix = "cls" + RegionPrefix Prefix = "rgn" OrgPrefix Prefix = "org" // Control plane prefixes diff --git a/svc/ctrl/api/certificate.go b/svc/ctrl/api/certificate.go index 3ef70383518..8b4b88b19a5 100644 --- a/svc/ctrl/api/certificate.go +++ b/svc/ctrl/api/certificate.go @@ -18,7 +18,6 @@ type certificateBootstrap struct { database db.Database defaultDomain string regionalDomain string - regions []string } // run bootstraps wildcard certificates for all configured domains and starts @@ -35,9 +34,14 @@ func (c *certificateBootstrap) run(ctx context.Context) { // Bootstrap per-region wildcards (e.g., *.us-west-2.aws.unkey.cloud) if c.regionalDomain != "" { - for _, region := range c.regions { - domain := fmt.Sprintf("*.%s.%s", region, c.regionalDomain) - c.bootstrapDomain(ctx, domain) + regions, err := db.Query.ListRegions(ctx, c.database.RO()) + if err != nil { + logger.Error("Failed to list regions for certificate bootstrap", "error", err) + } else { + for _, region := range regions { + domain := fmt.Sprintf("*.%s.%s", region.Name, c.regionalDomain) + c.bootstrapDomain(ctx, domain) + } } } } diff --git a/svc/ctrl/api/config.go b/svc/ctrl/api/config.go index 655f5814086..b576e9e032b 100644 --- a/svc/ctrl/api/config.go +++ b/svc/ctrl/api/config.go @@ -64,10 +64,6 @@ type Config struct { // Used by clients and services to authenticate with this control plane. AuthToken string `toml:"auth_token" config:"required,nonempty"` - // AvailableRegions is a list of available regions for deployments. - // Typically in the format "region.provider", ie "us-east-1.aws", "local.dev" - AvailableRegions []string `toml:"available_regions"` - // DefaultDomain is the fallback domain for system operations. // Used for wildcard certificate bootstrapping. When set, the API will // ensure a wildcard certificate exists for *.{DefaultDomain}. diff --git a/svc/ctrl/api/harness_test.go b/svc/ctrl/api/harness_test.go index e5980d43426..79d605202b3 100644 --- a/svc/ctrl/api/harness_test.go +++ b/svc/ctrl/api/harness_test.go @@ -93,14 +93,14 @@ func newWebhookHarness(t *testing.T, cfg webhookHarnessConfig) *webhookHarness { } apiConfig := Config{ - InstanceID: "test", - Region: "local", - HttpPort: ctrlPort, - PrometheusPort: 0, - AuthToken: "", - AvailableRegions: []string{"local.dev"}, - DefaultDomain: "", - RegionalDomain: "", + InstanceID: "test", + Region: "local", + HttpPort: ctrlPort, + PrometheusPort: 0, + AuthToken: "", + + DefaultDomain: "", + RegionalDomain: "", Database: config.DatabaseConfig{ Primary: mysqlCfg.DSN, ReadonlyReplica: "", diff --git a/svc/ctrl/api/run.go b/svc/ctrl/api/run.go index 35144a5eb74..83703ab0fe5 100644 --- a/svc/ctrl/api/run.go +++ b/svc/ctrl/api/run.go @@ -217,7 +217,6 @@ func Run(ctx context.Context, cfg Config) error { database: database, defaultDomain: cfg.DefaultDomain, regionalDomain: cfg.RegionalDomain, - regions: cfg.AvailableRegions, } r.Go(func(ctx context.Context) error { certBootstrap.run(ctx) diff --git a/svc/ctrl/integration/harness/harness.go b/svc/ctrl/integration/harness/harness.go index d2a9348843d..d7fbc8e1bd6 100644 --- a/svc/ctrl/integration/harness/harness.go +++ b/svc/ctrl/integration/harness/harness.go @@ -170,12 +170,12 @@ func New(t *testing.T) *Harness { }) deploySvc := deploy.New(deploy.Config{ - DB: database, - Clickhouse: chClient, - DefaultDomain: "test.example.com", - Vault: vaultClient, - SentinelImage: "test-sentinel:latest", - AvailableRegions: []string{"us-east-1"}, + DB: database, + Clickhouse: chClient, + DefaultDomain: "test.example.com", + Vault: vaultClient, + SentinelImage: "test-sentinel:latest", + GitHub: nil, DepotConfig: deploy.DepotConfig{APIUrl: "", ProjectRegion: ""}, RegistryConfig: deploy.RegistryConfig{URL: "", Username: "", Password: ""}, diff --git a/svc/ctrl/proto/ctrl/v1/cluster.proto b/svc/ctrl/proto/ctrl/v1/cluster.proto index b9cbadf1214..c5fc7c13dd3 100644 --- a/svc/ctrl/proto/ctrl/v1/cluster.proto +++ b/svc/ctrl/proto/ctrl/v1/cluster.proto @@ -66,7 +66,7 @@ service ClusterService { rpc GetDesiredCiliumNetworkPolicyState(GetDesiredCiliumNetworkPolicyStateRequest) returns (CiliumNetworkPolicyState); // Heartbeat is called periodically by krane agents to register their cluster - // and region with the control plane. This populates the cluster_regions and + // and region with the control plane. This populates the regions and // clusters tables, making regions dynamically discoverable. rpc Heartbeat(HeartbeatRequest) returns (HeartbeatResponse); } @@ -330,7 +330,7 @@ message DeleteDeployment { } // HeartbeatRequest is sent periodically by krane agents to register their -// presence. The control plane uses this to populate cluster_regions and +// presence. The control plane uses this to populate regions and // clusters tables. message HeartbeatRequest { // region is the geographic region name (e.g. "us-east-1"). diff --git a/svc/ctrl/services/cluster/BUILD.bazel b/svc/ctrl/services/cluster/BUILD.bazel index 572149f2de4..b16e6f0515f 100644 --- a/svc/ctrl/services/cluster/BUILD.bazel +++ b/svc/ctrl/services/cluster/BUILD.bazel @@ -8,6 +8,7 @@ go_library( "rpc_get_desired_cilium_network_policy_state.go", "rpc_get_desired_deployment_state.go", "rpc_get_desired_sentinel_state.go", + "rpc_heartbeat.go", "rpc_report_deployment_status.go", "rpc_report_sentinel_status.go", "rpc_watch_cilium_network_policies.go", diff --git a/svc/ctrl/services/cluster/rpc_heartbeat.go b/svc/ctrl/services/cluster/rpc_heartbeat.go new file mode 100644 index 00000000000..0457df52400 --- /dev/null +++ b/svc/ctrl/services/cluster/rpc_heartbeat.go @@ -0,0 +1,68 @@ +package cluster + +import ( + "context" + "time" + + "connectrpc.com/connect" + ctrlv1 "github.com/unkeyed/unkey/gen/proto/ctrl/v1" + "github.com/unkeyed/unkey/pkg/assert" + "github.com/unkeyed/unkey/pkg/db" + "github.com/unkeyed/unkey/pkg/logger" + "github.com/unkeyed/unkey/pkg/uid" +) + +// Heartbeat registers or refreshes a krane agent's cluster and region in the +// control plane. Agents call this periodically so the control plane knows which +// regions are available. +// +// The method upserts into regions (keyed by region name) and clusters +// (keyed by region_id), updating the heartbeat timestamp on each call. +func (s *Service) Heartbeat(ctx context.Context, req *connect.Request[ctrlv1.HeartbeatRequest]) (*connect.Response[ctrlv1.HeartbeatResponse], error) { + if err := s.authenticate(req); err != nil { + return nil, err + } + + region := req.Msg.GetRegion() + platform := req.Msg.GetPlatform() + + if err := assert.All( + assert.NotEmpty(region, "region is required"), + assert.NotEmpty(platform, "platform is required"), + ); err != nil { + return nil, err + } + + now := time.Now().UnixMilli() + + err := db.Query.UpsertRegion(ctx, s.db.RW(), db.UpsertRegionParams{ + ID: uid.New(uid.RegionPrefix), + Name: region, + Platform: platform, + }) + if err != nil { + logger.Error("failed to upsert region", "error", err, "region", region) + return nil, err + } + + regionID, err := db.Query.FindRegionByNameAndPlatform(ctx, s.db.RW(), db.FindRegionByNameAndPlatformParams{ + Name: region, + Platform: platform, + }) + if err != nil { + logger.Error("failed to find region", "error", err, "region", region) + return nil, err + } + + err = db.Query.UpsertCluster(ctx, s.db.RW(), db.UpsertClusterParams{ + ID: uid.New(uid.ClusterPrefix), + RegionID: regionID, + LastHeartbeatAt: uint64(now), + }) + if err != nil { + logger.Error("failed to upsert cluster", "error", err, "region", region) + return nil, err + } + + return connect.NewResponse(&ctrlv1.HeartbeatResponse{}), nil +} diff --git a/svc/ctrl/services/deployment/doc.go b/svc/ctrl/services/deployment/doc.go index 6e36b2aa511..bd5decc9889 100644 --- a/svc/ctrl/services/deployment/doc.go +++ b/svc/ctrl/services/deployment/doc.go @@ -45,7 +45,7 @@ // svc := deployment.New(deployment.Config{ // Database: db, // Restate: restateClient, -// AvailableRegions: []string{"us-east-1", "eu-west-1"}, + // BuildStorage: s3Storage, // }) package deployment diff --git a/svc/ctrl/worker/config.go b/svc/ctrl/worker/config.go index e131f2204e1..5c8ef10b5e3 100644 --- a/svc/ctrl/worker/config.go +++ b/svc/ctrl/worker/config.go @@ -212,10 +212,6 @@ type Config struct { // Overrides default sentinel image with custom build or registry. SentinelImage string `toml:"sentinel_image" config:"default=ghcr.io/unkeyed/unkey:local"` - // AvailableRegions is a list of available regions for deployments. - // typically in the format "region.provider", ie "us-east-1.aws", "local.dev" - AvailableRegions []string `toml:"available_regions"` - // CnameDomain is the base domain for custom domain CNAME targets. // Each custom domain gets a unique subdomain like "{random}.{CnameDomain}". CnameDomain string `toml:"cname_domain" config:"required,nonempty"` diff --git a/svc/ctrl/worker/deploy/deploy_handler.go b/svc/ctrl/worker/deploy/deploy_handler.go index dccd6fe319c..c0f46d3b56a 100644 --- a/svc/ctrl/worker/deploy/deploy_handler.go +++ b/svc/ctrl/worker/deploy/deploy_handler.go @@ -375,7 +375,7 @@ func (w *Workflow) buildImage(ctx restate.ObjectContext, req *hydrav1.DeployRequ // // Region selection uses the environment's runtime settings: if a region config is // present, only those regions are used with their configured replica counts; -// otherwise all of [Workflow.availableRegions] are used with 1 replica each. +// otherwise the deployment fails with a terminal error. // // createTopologies also registers compensations for every inserted // topology. Compensation deletes by deployment, region, and version so retries @@ -408,13 +408,16 @@ func (w *Workflow) createTopologies( } } - var regions []string if len(regionConfig) == 0 { - regions = w.availableRegions - } else { - for r := range regionConfig { - regions = append(regions, r) - } + return nil, fault.Wrap( + restate.TerminalError(fmt.Errorf("no regions configured for app %s in environment %s", deployment.AppID, deployment.EnvironmentID), 400), + fault.Public("No regions configured. Please configure at least one region before deploying."), + ) + } + + regions := make([]string, 0, len(regionConfig)) + for r := range regionConfig { + regions = append(regions, r) } topologies := make([]db.InsertDeploymentTopologyParams, 0, len(regions)) diff --git a/svc/ctrl/worker/deploy/service.go b/svc/ctrl/worker/deploy/service.go index 2f662406702..7d08be4c1a1 100644 --- a/svc/ctrl/worker/deploy/service.go +++ b/svc/ctrl/worker/deploy/service.go @@ -42,11 +42,11 @@ type Workflow struct { hydrav1.UnimplementedDeployServiceServer db db.Database - defaultDomain string - vault vault.VaultServiceClient - sentinelImage string - availableRegions []string - github githubclient.GitHubClient + defaultDomain string + vault vault.VaultServiceClient + sentinelImage string + + github githubclient.GitHubClient // Build dependencies depotConfig DepotConfig @@ -72,9 +72,6 @@ type Config struct { // SentinelImage is the Docker image used for sentinel containers. SentinelImage string - // AvailableRegions is the list of available regions for deployments. - AvailableRegions []string - // GitHub provides access to GitHub API for downloading tarballs. GitHub githubclient.GitHubClient @@ -103,12 +100,12 @@ func New(cfg Config) *Workflow { defaultDomain: cfg.DefaultDomain, vault: cfg.Vault, sentinelImage: cfg.SentinelImage, - availableRegions: cfg.AvailableRegions, - github: cfg.GitHub, - depotConfig: cfg.DepotConfig, - registryConfig: cfg.RegistryConfig, - buildPlatform: cfg.BuildPlatform, - clickhouse: cfg.Clickhouse, - allowUnauthenticatedDeployments: cfg.AllowUnauthenticatedDeployments, + + github: cfg.GitHub, + depotConfig: cfg.DepotConfig, + registryConfig: cfg.RegistryConfig, + buildPlatform: cfg.BuildPlatform, + clickhouse: cfg.Clickhouse, + allowUnauthenticatedDeployments: cfg.AllowUnauthenticatedDeployments, } } diff --git a/svc/ctrl/worker/run.go b/svc/ctrl/worker/run.go index 32f991089ba..b2d525cb372 100644 --- a/svc/ctrl/worker/run.go +++ b/svc/ctrl/worker/run.go @@ -161,11 +161,11 @@ func Run(ctx context.Context, cfg Config) error { } restateSrv.Bind(hydrav1.NewDeployServiceServer(deploy.New(deploy.Config{ - DB: database, - DefaultDomain: cfg.DefaultDomain, - Vault: vaultClient, - SentinelImage: cfg.SentinelImage, - AvailableRegions: cfg.AvailableRegions, + DB: database, + DefaultDomain: cfg.DefaultDomain, + Vault: vaultClient, + SentinelImage: cfg.SentinelImage, + GitHub: ghClient, RegistryConfig: deploy.RegistryConfig(cfg.GetRegistryConfig()), BuildPlatform: deploy.BuildPlatform(buildPlatform), diff --git a/svc/krane/BUILD.bazel b/svc/krane/BUILD.bazel index 8b100c4d494..fe58e784e59 100644 --- a/svc/krane/BUILD.bazel +++ b/svc/krane/BUILD.bazel @@ -18,8 +18,8 @@ go_library( "//pkg/config", "//pkg/logger", "//pkg/otel", - "//pkg/repeat", "//pkg/prometheus", + "//pkg/repeat", "//pkg/rpc/interceptor", "//pkg/runner", "//pkg/version", diff --git a/svc/krane/internal/testutil/mock_cluster_client.go b/svc/krane/internal/testutil/mock_cluster_client.go index 7b45983080e..787e914de17 100644 --- a/svc/krane/internal/testutil/mock_cluster_client.go +++ b/svc/krane/internal/testutil/mock_cluster_client.go @@ -26,7 +26,7 @@ type MockClusterClient struct { GetDesiredDeploymentStateFunc func(context.Context, *ctrlv1.GetDesiredDeploymentStateRequest) (*ctrlv1.DeploymentState, error) ReportDeploymentStatusFunc func(context.Context, *ctrlv1.ReportDeploymentStatusRequest) (*ctrlv1.ReportDeploymentStatusResponse, error) GetDesiredCiliumNetworkPolicyStateFunc func(context.Context, *ctrlv1.GetDesiredCiliumNetworkPolicyStateRequest) (*ctrlv1.CiliumNetworkPolicyState, error) - HeartbeatFunc func(context.Context, *ctrlv1.HeartbeatRequest) (*ctrlv1.HeartbeatResponse, error) + HeartbeatFunc func(context.Context, *ctrlv1.HeartbeatRequest) (*ctrlv1.HeartbeatResponse, error) ReportDeploymentStatusCalls []*ctrlv1.ReportDeploymentStatusRequest ReportSentinelStatusCalls []*ctrlv1.ReportSentinelStatusRequest } diff --git a/svc/krane/run.go b/svc/krane/run.go index e9124e3968d..6c13ad618f6 100644 --- a/svc/krane/run.go +++ b/svc/krane/run.go @@ -15,9 +15,9 @@ import ( "github.com/unkeyed/unkey/gen/proto/vault/v1/vaultv1connect" "github.com/unkeyed/unkey/gen/rpc/vault" "github.com/unkeyed/unkey/pkg/logger" - "github.com/unkeyed/unkey/pkg/repeat" "github.com/unkeyed/unkey/pkg/otel" "github.com/unkeyed/unkey/pkg/prometheus" + "github.com/unkeyed/unkey/pkg/repeat" "github.com/unkeyed/unkey/pkg/rpc/interceptor" "github.com/unkeyed/unkey/pkg/runner" pkgversion "github.com/unkeyed/unkey/pkg/version" diff --git a/web/apps/dashboard/gen/proto/ctrl/v1/cluster_pb.ts b/web/apps/dashboard/gen/proto/ctrl/v1/cluster_pb.ts index 17f8b10517e..d5b9c688a25 100644 --- a/web/apps/dashboard/gen/proto/ctrl/v1/cluster_pb.ts +++ b/web/apps/dashboard/gen/proto/ctrl/v1/cluster_pb.ts @@ -814,7 +814,7 @@ export const DeleteDeploymentSchema: GenMessage = /*@__PURE__* /** * HeartbeatRequest is sent periodically by krane agents to register their - * presence. The control plane uses this to populate cluster_regions and + * presence. The control plane uses this to populate regions and * clusters tables. * * @generated from message ctrl.v1.HeartbeatRequest @@ -994,7 +994,7 @@ export const ClusterService: GenService<{ }, /** * Heartbeat is called periodically by krane agents to register their cluster - * and region with the control plane. This populates the cluster_regions and + * and region with the control plane. This populates the regions and * clusters tables, making regions dynamically discoverable. * * @generated from rpc ctrl.v1.ClusterService.Heartbeat diff --git a/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/get-available-regions.ts b/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/get-available-regions.ts index d08d24fc137..4dd66742248 100644 --- a/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/get-available-regions.ts +++ b/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/get-available-regions.ts @@ -2,8 +2,8 @@ import { db } from "@/lib/db"; import { workspaceProcedure } from "../../../trpc"; export const getAvailableRegions = workspaceProcedure.query(async () => { - const regions = await db.query.clusterRegions.findMany({ + const r = await db.query.regions.findMany({ columns: { id: true, name: true }, }); - return regions; + return r; }); diff --git a/web/internal/db/src/schema/clusters.ts b/web/internal/db/src/schema/clusters.ts index ee68f8c0ab8..f3a150465df 100644 --- a/web/internal/db/src/schema/clusters.ts +++ b/web/internal/db/src/schema/clusters.ts @@ -1,6 +1,6 @@ import { relations } from "drizzle-orm"; import { bigint, mysqlTable, varchar } from "drizzle-orm/mysql-core"; -import { clusterRegions } from "./cluster_regions"; +import { regions } from "./regions"; // clusters tracks our kubernetes clusters // each krane instance will heartbeat against the control plane, which then writes to this table @@ -10,7 +10,7 @@ export const clusters = mysqlTable("clusters", { pk: bigint("pk", { mode: "number", unsigned: true }).autoincrement().primaryKey(), id: varchar("id", { length: 64 }).notNull().unique(), - regionId: varchar("region_id", { length: 64 }).notNull(), + regionId: varchar("region_id", { length: 64 }).notNull().unique(), lastHeartbeatAt: bigint("last_heartbeat_at", { mode: "number", @@ -19,8 +19,8 @@ export const clusters = mysqlTable("clusters", { }); export const regionsRelations = relations(clusters, ({ one }) => ({ - workspace: one(clusterRegions, { + workspace: one(regions, { fields: [clusters.regionId], - references: [clusterRegions.id], + references: [regions.id], }), })); diff --git a/web/internal/db/src/schema/index.ts b/web/internal/db/src/schema/index.ts index 764274651cf..cfa33e5041d 100644 --- a/web/internal/db/src/schema/index.ts +++ b/web/internal/db/src/schema/index.ts @@ -36,5 +36,5 @@ export * from "./github_app"; export * from "./cilium"; export * from "./clusters"; -export * from "./cluster_regions"; +export * from "./regions"; export * from "./horizontal_autoscaling_policies"; diff --git a/web/internal/db/src/schema/cluster_regions.ts b/web/internal/db/src/schema/regions.ts similarity index 73% rename from web/internal/db/src/schema/cluster_regions.ts rename to web/internal/db/src/schema/regions.ts index d9423349770..53c9398e702 100644 --- a/web/internal/db/src/schema/cluster_regions.ts +++ b/web/internal/db/src/schema/regions.ts @@ -2,8 +2,8 @@ import { relations } from "drizzle-orm"; import { bigint, mysqlTable, uniqueIndex, varchar } from "drizzle-orm/mysql-core"; import { clusters } from "./clusters"; -export const clusterRegions = mysqlTable( - "cluster_regions", +export const regions = mysqlTable( + "regions", { pk: bigint("pk", { mode: "number", unsigned: true }).autoincrement().primaryKey(), id: varchar("id", { length: 64 }).notNull().unique(), @@ -12,9 +12,9 @@ export const clusterRegions = mysqlTable( // e.g. aws, gcp, azure, local, etc. platform: varchar("platform", { length: 64 }).notNull(), }, - (table) => [uniqueIndex("unique_reqion_per_platform").on(table.name, table.platform)], + (table) => [uniqueIndex("unique_region_per_platform").on(table.name, table.platform)], ); -export const clusterRegionRelations = relations(clusterRegions, ({ many }) => ({ +export const regionRelations = relations(regions, ({ many }) => ({ clusters: many(clusters), })); From 895ae701067e516ce19294fbea3ca444912558f3 Mon Sep 17 00:00:00 2001 From: chronark Date: Sat, 7 Mar 2026 18:06:33 +0100 Subject: [PATCH 16/24] feat: deploy workflow reads from appRegionalSettings, add region_id to deployment_topology - Deploy handler now reads regions from app_regional_settings table instead of deprecated regionConfig on app_runtime_settings - Fails with terminal error if no regions are configured - Added region_id column to deployment_topology table - Added FindAppRegionalSettingsByAppAndEnv SQL query with regions join - Updated integration harness to create regions before topologies --- pkg/db/BUILD.bazel | 1 + ...tings_find_by_app_and_env.sql_generated.go | 65 +++++++++++++++++++ ...eployment_topology_insert.sql_generated.go | 5 +- ...eployment_topology_insert.sql_generated.go | 6 ++ ...topology_list_by_versions.sql_generated.go | 5 +- ...ent_topology_list_desired.sql_generated.go | 5 +- pkg/db/models_generated.go | 1 + pkg/db/querier_generated.go | 17 ++++- ..._regional_settings_find_by_app_and_env.sql | 9 +++ pkg/db/queries/deployment_topology_insert.sql | 2 + pkg/db/schema.sql | 1 + svc/ctrl/integration/harness.go | 17 +++++ svc/ctrl/worker/deploy/deploy_handler.go | 40 ++++-------- .../db/src/schema/deployment_topology.ts | 1 + 14 files changed, 140 insertions(+), 35 deletions(-) create mode 100644 pkg/db/app_regional_settings_find_by_app_and_env.sql_generated.go create mode 100644 pkg/db/queries/app_regional_settings_find_by_app_and_env.sql diff --git a/pkg/db/BUILD.bazel b/pkg/db/BUILD.bazel index 49fcb059158..b19494ad0ce 100644 --- a/pkg/db/BUILD.bazel +++ b/pkg/db/BUILD.bazel @@ -34,6 +34,7 @@ go_library( "app_find_with_settings.sql_generated.go", "app_insert.sql_generated.go", "app_list_by_project.sql_generated.go", + "app_regional_settings_find_by_app_and_env.sql_generated.go", "app_runtime_settings_find.sql_generated.go", "app_runtime_settings_upsert.sql_generated.go", "app_update_deployments.sql_generated.go", diff --git a/pkg/db/app_regional_settings_find_by_app_and_env.sql_generated.go b/pkg/db/app_regional_settings_find_by_app_and_env.sql_generated.go new file mode 100644 index 00000000000..6cbefa9999f --- /dev/null +++ b/pkg/db/app_regional_settings_find_by_app_and_env.sql_generated.go @@ -0,0 +1,65 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.30.0 +// source: app_regional_settings_find_by_app_and_env.sql + +package db + +import ( + "context" +) + +const findAppRegionalSettingsByAppAndEnv = `-- name: FindAppRegionalSettingsByAppAndEnv :many +SELECT + ars.region_id, + r.name AS region_name, + ars.replicas +FROM app_regional_settings ars +JOIN regions r ON r.id = ars.region_id +WHERE ars.app_id = ? + AND ars.environment_id = ? +` + +type FindAppRegionalSettingsByAppAndEnvParams struct { + AppID string `db:"app_id"` + EnvironmentID string `db:"environment_id"` +} + +type FindAppRegionalSettingsByAppAndEnvRow struct { + RegionID string `db:"region_id"` + RegionName string `db:"region_name"` + Replicas int32 `db:"replicas"` +} + +// FindAppRegionalSettingsByAppAndEnv +// +// SELECT +// ars.region_id, +// r.name AS region_name, +// ars.replicas +// FROM app_regional_settings ars +// JOIN regions r ON r.id = ars.region_id +// WHERE ars.app_id = ? +// AND ars.environment_id = ? +func (q *Queries) FindAppRegionalSettingsByAppAndEnv(ctx context.Context, db DBTX, arg FindAppRegionalSettingsByAppAndEnvParams) ([]FindAppRegionalSettingsByAppAndEnvRow, error) { + rows, err := db.QueryContext(ctx, findAppRegionalSettingsByAppAndEnv, arg.AppID, arg.EnvironmentID) + if err != nil { + return nil, err + } + defer rows.Close() + var items []FindAppRegionalSettingsByAppAndEnvRow + for rows.Next() { + var i FindAppRegionalSettingsByAppAndEnvRow + if err := rows.Scan(&i.RegionID, &i.RegionName, &i.Replicas); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} diff --git a/pkg/db/bulk_deployment_topology_insert.sql_generated.go b/pkg/db/bulk_deployment_topology_insert.sql_generated.go index 47d2f8d66fc..49b4a1ccf03 100644 --- a/pkg/db/bulk_deployment_topology_insert.sql_generated.go +++ b/pkg/db/bulk_deployment_topology_insert.sql_generated.go @@ -9,7 +9,7 @@ import ( ) // bulkInsertDeploymentTopology is the base query for bulk insert -const bulkInsertDeploymentTopology = `INSERT INTO ` + "`" + `deployment_topology` + "`" + ` ( workspace_id, deployment_id, region, desired_replicas, desired_status, version, created_at ) VALUES %s` +const bulkInsertDeploymentTopology = `INSERT INTO ` + "`" + `deployment_topology` + "`" + ` ( workspace_id, deployment_id, region, region_id, desired_replicas, desired_status, version, created_at ) VALUES %s` // InsertDeploymentTopologies performs bulk insert in a single query func (q *BulkQueries) InsertDeploymentTopologies(ctx context.Context, db DBTX, args []InsertDeploymentTopologyParams) error { @@ -21,7 +21,7 @@ func (q *BulkQueries) InsertDeploymentTopologies(ctx context.Context, db DBTX, a // Build the bulk insert query valueClauses := make([]string, len(args)) for i := range args { - valueClauses[i] = "( ?, ?, ?, ?, ?, ?, ? )" + valueClauses[i] = "( ?, ?, ?, ?, ?, ?, ?, ? )" } bulkQuery := fmt.Sprintf(bulkInsertDeploymentTopology, strings.Join(valueClauses, ", ")) @@ -32,6 +32,7 @@ func (q *BulkQueries) InsertDeploymentTopologies(ctx context.Context, db DBTX, a allArgs = append(allArgs, arg.WorkspaceID) allArgs = append(allArgs, arg.DeploymentID) allArgs = append(allArgs, arg.Region) + allArgs = append(allArgs, arg.RegionID) allArgs = append(allArgs, arg.DesiredReplicas) allArgs = append(allArgs, arg.DesiredStatus) allArgs = append(allArgs, arg.Version) diff --git a/pkg/db/deployment_topology_insert.sql_generated.go b/pkg/db/deployment_topology_insert.sql_generated.go index 1ab6dc8a1f1..15ea4b561e8 100644 --- a/pkg/db/deployment_topology_insert.sql_generated.go +++ b/pkg/db/deployment_topology_insert.sql_generated.go @@ -14,6 +14,7 @@ INSERT INTO ` + "`" + `deployment_topology` + "`" + ` ( workspace_id, deployment_id, region, + region_id, desired_replicas, desired_status, version, @@ -25,6 +26,7 @@ INSERT INTO ` + "`" + `deployment_topology` + "`" + ` ( ?, ?, ?, + ?, ? ) ` @@ -33,6 +35,7 @@ type InsertDeploymentTopologyParams struct { WorkspaceID string `db:"workspace_id"` DeploymentID string `db:"deployment_id"` Region string `db:"region"` + RegionID string `db:"region_id"` DesiredReplicas int32 `db:"desired_replicas"` DesiredStatus DeploymentTopologyDesiredStatus `db:"desired_status"` Version uint64 `db:"version"` @@ -45,6 +48,7 @@ type InsertDeploymentTopologyParams struct { // workspace_id, // deployment_id, // region, +// region_id, // desired_replicas, // desired_status, // version, @@ -56,6 +60,7 @@ type InsertDeploymentTopologyParams struct { // ?, // ?, // ?, +// ?, // ? // ) func (q *Queries) InsertDeploymentTopology(ctx context.Context, db DBTX, arg InsertDeploymentTopologyParams) error { @@ -63,6 +68,7 @@ func (q *Queries) InsertDeploymentTopology(ctx context.Context, db DBTX, arg Ins arg.WorkspaceID, arg.DeploymentID, arg.Region, + arg.RegionID, arg.DesiredReplicas, arg.DesiredStatus, arg.Version, diff --git a/pkg/db/deployment_topology_list_by_versions.sql_generated.go b/pkg/db/deployment_topology_list_by_versions.sql_generated.go index 680b3fb13e4..b9f1577a2ff 100644 --- a/pkg/db/deployment_topology_list_by_versions.sql_generated.go +++ b/pkg/db/deployment_topology_list_by_versions.sql_generated.go @@ -12,7 +12,7 @@ import ( const listDeploymentTopologyByRegion = `-- name: ListDeploymentTopologyByRegion :many SELECT - dt.pk, dt.workspace_id, dt.deployment_id, dt.region, dt.desired_replicas, dt.version, dt.desired_status, dt.created_at, dt.updated_at, + dt.pk, dt.workspace_id, dt.deployment_id, dt.region, dt.region_id, dt.desired_replicas, dt.version, dt.desired_status, dt.created_at, dt.updated_at, d.pk, d.id, d.k8s_name, d.workspace_id, d.project_id, d.environment_id, d.app_id, d.image, d.build_id, d.git_commit_sha, d.git_branch, d.git_commit_message, d.git_commit_author_handle, d.git_commit_author_avatar_url, d.git_commit_timestamp, d.sentinel_config, d.openapi_spec, d.cpu_millicores, d.memory_mib, d.desired_state, d.encrypted_environment_variables, d.command, d.port, d.shutdown_signal, d.healthcheck, d.status, d.created_at, d.updated_at, w.k8s_namespace FROM ` + "`" + `deployment_topology` + "`" + ` dt @@ -39,7 +39,7 @@ type ListDeploymentTopologyByRegionRow struct { // Used by WatchDeployments to stream deployment state changes to krane agents. // // SELECT -// dt.pk, dt.workspace_id, dt.deployment_id, dt.region, dt.desired_replicas, dt.version, dt.desired_status, dt.created_at, dt.updated_at, +// dt.pk, dt.workspace_id, dt.deployment_id, dt.region, dt.region_id, dt.desired_replicas, dt.version, dt.desired_status, dt.created_at, dt.updated_at, // d.pk, d.id, d.k8s_name, d.workspace_id, d.project_id, d.environment_id, d.app_id, d.image, d.build_id, d.git_commit_sha, d.git_branch, d.git_commit_message, d.git_commit_author_handle, d.git_commit_author_avatar_url, d.git_commit_timestamp, d.sentinel_config, d.openapi_spec, d.cpu_millicores, d.memory_mib, d.desired_state, d.encrypted_environment_variables, d.command, d.port, d.shutdown_signal, d.healthcheck, d.status, d.created_at, d.updated_at, // w.k8s_namespace // FROM `deployment_topology` dt @@ -62,6 +62,7 @@ func (q *Queries) ListDeploymentTopologyByRegion(ctx context.Context, db DBTX, a &i.DeploymentTopology.WorkspaceID, &i.DeploymentTopology.DeploymentID, &i.DeploymentTopology.Region, + &i.DeploymentTopology.RegionID, &i.DeploymentTopology.DesiredReplicas, &i.DeploymentTopology.Version, &i.DeploymentTopology.DesiredStatus, diff --git a/pkg/db/deployment_topology_list_desired.sql_generated.go b/pkg/db/deployment_topology_list_desired.sql_generated.go index 23524e3764b..5a699812302 100644 --- a/pkg/db/deployment_topology_list_desired.sql_generated.go +++ b/pkg/db/deployment_topology_list_desired.sql_generated.go @@ -12,7 +12,7 @@ import ( const listDesiredDeploymentTopology = `-- name: ListDesiredDeploymentTopology :many SELECT - dt.pk, dt.workspace_id, dt.deployment_id, dt.region, dt.desired_replicas, dt.version, dt.desired_status, dt.created_at, dt.updated_at, + dt.pk, dt.workspace_id, dt.deployment_id, dt.region, dt.region_id, dt.desired_replicas, dt.version, dt.desired_status, dt.created_at, dt.updated_at, d.pk, d.id, d.k8s_name, d.workspace_id, d.project_id, d.environment_id, d.app_id, d.image, d.build_id, d.git_commit_sha, d.git_branch, d.git_commit_message, d.git_commit_author_handle, d.git_commit_author_avatar_url, d.git_commit_timestamp, d.sentinel_config, d.openapi_spec, d.cpu_millicores, d.memory_mib, d.desired_state, d.encrypted_environment_variables, d.command, d.port, d.shutdown_signal, d.healthcheck, d.status, d.created_at, d.updated_at, w.k8s_namespace FROM ` + "`" + `deployment_topology` + "`" + ` dt @@ -42,7 +42,7 @@ type ListDesiredDeploymentTopologyRow struct { // Used during bootstrap to stream all running deployments to krane. // // SELECT -// dt.pk, dt.workspace_id, dt.deployment_id, dt.region, dt.desired_replicas, dt.version, dt.desired_status, dt.created_at, dt.updated_at, +// dt.pk, dt.workspace_id, dt.deployment_id, dt.region, dt.region_id, dt.desired_replicas, dt.version, dt.desired_status, dt.created_at, dt.updated_at, // d.pk, d.id, d.k8s_name, d.workspace_id, d.project_id, d.environment_id, d.app_id, d.image, d.build_id, d.git_commit_sha, d.git_branch, d.git_commit_message, d.git_commit_author_handle, d.git_commit_author_avatar_url, d.git_commit_timestamp, d.sentinel_config, d.openapi_spec, d.cpu_millicores, d.memory_mib, d.desired_state, d.encrypted_environment_variables, d.command, d.port, d.shutdown_signal, d.healthcheck, d.status, d.created_at, d.updated_at, // w.k8s_namespace // FROM `deployment_topology` dt @@ -73,6 +73,7 @@ func (q *Queries) ListDesiredDeploymentTopology(ctx context.Context, db DBTX, ar &i.DeploymentTopology.WorkspaceID, &i.DeploymentTopology.DeploymentID, &i.DeploymentTopology.Region, + &i.DeploymentTopology.RegionID, &i.DeploymentTopology.DesiredReplicas, &i.DeploymentTopology.Version, &i.DeploymentTopology.DesiredStatus, diff --git a/pkg/db/models_generated.go b/pkg/db/models_generated.go index 87ca16e47de..fbbe64ab9c3 100644 --- a/pkg/db/models_generated.go +++ b/pkg/db/models_generated.go @@ -1197,6 +1197,7 @@ type DeploymentTopology struct { WorkspaceID string `db:"workspace_id"` DeploymentID string `db:"deployment_id"` Region string `db:"region"` + RegionID string `db:"region_id"` DesiredReplicas int32 `db:"desired_replicas"` Version uint64 `db:"version"` DesiredStatus DeploymentTopologyDesiredStatus `db:"desired_status"` diff --git a/pkg/db/querier_generated.go b/pkg/db/querier_generated.go index b9209de06b6..0f37a8080c8 100644 --- a/pkg/db/querier_generated.go +++ b/pkg/db/querier_generated.go @@ -209,6 +209,17 @@ type Querier interface { // WHERE app_id = ? // AND environment_id = ? FindAppEnvVarsByAppAndEnv(ctx context.Context, db DBTX, arg FindAppEnvVarsByAppAndEnvParams) ([]FindAppEnvVarsByAppAndEnvRow, error) + //FindAppRegionalSettingsByAppAndEnv + // + // SELECT + // ars.region_id, + // r.name AS region_name, + // ars.replicas + // FROM app_regional_settings ars + // JOIN regions r ON r.id = ars.region_id + // WHERE ars.app_id = ? + // AND ars.environment_id = ? + FindAppRegionalSettingsByAppAndEnv(ctx context.Context, db DBTX, arg FindAppRegionalSettingsByAppAndEnvParams) ([]FindAppRegionalSettingsByAppAndEnvRow, error) //FindAppRuntimeSettingsByAppAndEnv // // SELECT app_runtime_settings.pk, app_runtime_settings.workspace_id, app_runtime_settings.app_id, app_runtime_settings.environment_id, app_runtime_settings.port, app_runtime_settings.cpu_millicores, app_runtime_settings.memory_mib, app_runtime_settings.command, app_runtime_settings.healthcheck, app_runtime_settings.region_config, app_runtime_settings.shutdown_signal, app_runtime_settings.sentinel_config, app_runtime_settings.created_at, app_runtime_settings.updated_at @@ -1416,6 +1427,7 @@ type Querier interface { // workspace_id, // deployment_id, // region, + // region_id, // desired_replicas, // desired_status, // version, @@ -1427,6 +1439,7 @@ type Querier interface { // ?, // ?, // ?, + // ?, // ? // ) InsertDeploymentTopology(ctx context.Context, db DBTX, arg InsertDeploymentTopologyParams) error @@ -1895,7 +1908,7 @@ type Querier interface { // Used by WatchDeployments to stream deployment state changes to krane agents. // // SELECT - // dt.pk, dt.workspace_id, dt.deployment_id, dt.region, dt.desired_replicas, dt.version, dt.desired_status, dt.created_at, dt.updated_at, + // dt.pk, dt.workspace_id, dt.deployment_id, dt.region, dt.region_id, dt.desired_replicas, dt.version, dt.desired_status, dt.created_at, dt.updated_at, // d.pk, d.id, d.k8s_name, d.workspace_id, d.project_id, d.environment_id, d.app_id, d.image, d.build_id, d.git_commit_sha, d.git_branch, d.git_commit_message, d.git_commit_author_handle, d.git_commit_author_avatar_url, d.git_commit_timestamp, d.sentinel_config, d.openapi_spec, d.cpu_millicores, d.memory_mib, d.desired_state, d.encrypted_environment_variables, d.command, d.port, d.shutdown_signal, d.healthcheck, d.status, d.created_at, d.updated_at, // w.k8s_namespace // FROM `deployment_topology` dt @@ -1917,7 +1930,7 @@ type Querier interface { // Used during bootstrap to stream all running deployments to krane. // // SELECT - // dt.pk, dt.workspace_id, dt.deployment_id, dt.region, dt.desired_replicas, dt.version, dt.desired_status, dt.created_at, dt.updated_at, + // dt.pk, dt.workspace_id, dt.deployment_id, dt.region, dt.region_id, dt.desired_replicas, dt.version, dt.desired_status, dt.created_at, dt.updated_at, // d.pk, d.id, d.k8s_name, d.workspace_id, d.project_id, d.environment_id, d.app_id, d.image, d.build_id, d.git_commit_sha, d.git_branch, d.git_commit_message, d.git_commit_author_handle, d.git_commit_author_avatar_url, d.git_commit_timestamp, d.sentinel_config, d.openapi_spec, d.cpu_millicores, d.memory_mib, d.desired_state, d.encrypted_environment_variables, d.command, d.port, d.shutdown_signal, d.healthcheck, d.status, d.created_at, d.updated_at, // w.k8s_namespace // FROM `deployment_topology` dt diff --git a/pkg/db/queries/app_regional_settings_find_by_app_and_env.sql b/pkg/db/queries/app_regional_settings_find_by_app_and_env.sql new file mode 100644 index 00000000000..e31b29469c7 --- /dev/null +++ b/pkg/db/queries/app_regional_settings_find_by_app_and_env.sql @@ -0,0 +1,9 @@ +-- name: FindAppRegionalSettingsByAppAndEnv :many +SELECT + ars.region_id, + r.name AS region_name, + ars.replicas +FROM app_regional_settings ars +JOIN regions r ON r.id = ars.region_id +WHERE ars.app_id = sqlc.arg(app_id) + AND ars.environment_id = sqlc.arg(environment_id); diff --git a/pkg/db/queries/deployment_topology_insert.sql b/pkg/db/queries/deployment_topology_insert.sql index 37158bb00ba..0f416dd05de 100644 --- a/pkg/db/queries/deployment_topology_insert.sql +++ b/pkg/db/queries/deployment_topology_insert.sql @@ -3,6 +3,7 @@ INSERT INTO `deployment_topology` ( workspace_id, deployment_id, region, + region_id, desired_replicas, desired_status, version, @@ -11,6 +12,7 @@ INSERT INTO `deployment_topology` ( sqlc.arg(workspace_id), sqlc.arg(deployment_id), sqlc.arg(region), + sqlc.arg(region_id), sqlc.arg(desired_replicas), sqlc.arg(desired_status), sqlc.arg(version), diff --git a/pkg/db/schema.sql b/pkg/db/schema.sql index ce8179283f5..a497603b19f 100644 --- a/pkg/db/schema.sql +++ b/pkg/db/schema.sql @@ -535,6 +535,7 @@ CREATE TABLE `deployment_topology` ( `workspace_id` varchar(64) NOT NULL, `deployment_id` varchar(64) NOT NULL, `region` varchar(64) NOT NULL, + `region_id` varchar(64) NOT NULL DEFAULT '', `desired_replicas` int NOT NULL, `version` bigint unsigned NOT NULL, `desired_status` enum('starting','started','stopping','stopped') NOT NULL, diff --git a/svc/ctrl/integration/harness.go b/svc/ctrl/integration/harness.go index 3bce686de2c..bd8eb887509 100644 --- a/svc/ctrl/integration/harness.go +++ b/svc/ctrl/integration/harness.go @@ -157,11 +157,27 @@ func (h *Harness) CreateDeployment(ctx context.Context, req CreateDeploymentRequ _, err = h.DB.RW().ExecContext(ctx, "UPDATE deployments SET image = ? WHERE id = ?", "nginx:1.19", deploymentID) require.NoError(h.t, err) + // Ensure the region exists + regionID := uid.New(uid.RegionPrefix) + err = db.Query.UpsertRegion(ctx, h.DB.RW(), db.UpsertRegionParams{ + ID: regionID, + Name: req.Region, + Platform: "test", + }) + require.NoError(h.t, err) + + regionID, err = db.Query.FindRegionByNameAndPlatform(ctx, h.DB.RO(), db.FindRegionByNameAndPlatformParams{ + Name: req.Region, + Platform: "test", + }) + require.NoError(h.t, err) + h.versionCounter++ err = db.Query.InsertDeploymentTopology(ctx, h.DB.RW(), db.InsertDeploymentTopologyParams{ WorkspaceID: workspaceID, DeploymentID: deploymentID, Region: req.Region, + RegionID: regionID, DesiredReplicas: 1, DesiredStatus: db.DeploymentTopologyDesiredStatusStarted, Version: h.versionCounter, @@ -179,6 +195,7 @@ func (h *Harness) CreateDeployment(ctx context.Context, req CreateDeploymentRequ WorkspaceID: workspaceID, DeploymentID: deploymentID, Region: req.Region, + RegionID: regionID, DesiredReplicas: 1, DesiredStatus: db.DeploymentTopologyDesiredStatusStarted, Version: h.versionCounter, diff --git a/svc/ctrl/worker/deploy/deploy_handler.go b/svc/ctrl/worker/deploy/deploy_handler.go index c0f46d3b56a..dda9ee21059 100644 --- a/svc/ctrl/worker/deploy/deploy_handler.go +++ b/svc/ctrl/worker/deploy/deploy_handler.go @@ -386,44 +386,32 @@ func (w *Workflow) createTopologies( workspace db.Workspace, deployment db.Deployment, ) ([]db.InsertDeploymentTopologyParams, error) { - // Read region config from runtime settings to determine per-region replica counts. - // If regionConfig is empty, deploy to all available regions with 1 replica each (default). - // If regionConfig has entries, only deploy to those regions with the specified counts. - regionConfig := map[string]int{} - runtimeSettings, err := restate.Run(ctx, func(runCtx restate.RunContext) (db.FindAppRuntimeSettingsByAppAndEnvRow, error) { - return db.Query.FindAppRuntimeSettingsByAppAndEnv(runCtx, w.db.RO(), db.FindAppRuntimeSettingsByAppAndEnvParams{ + // Read regional settings to determine per-region replica counts. + // If no regional settings exist, fail with a terminal error. + regionalSettings, err := restate.Run(ctx, func(runCtx restate.RunContext) ([]db.FindAppRegionalSettingsByAppAndEnvRow, error) { + return db.Query.FindAppRegionalSettingsByAppAndEnv(runCtx, w.db.RO(), db.FindAppRegionalSettingsByAppAndEnvParams{ AppID: deployment.AppID, EnvironmentID: deployment.EnvironmentID, }) - }, restate.WithName("find runtime settings for region config")) + }, restate.WithName("find regional settings")) if err != nil { return nil, fault.Wrap( - fmt.Errorf("failed to find runtime settings for environment %s: %w", deployment.EnvironmentID, err), + fmt.Errorf("failed to find regional settings for environment %s: %w", deployment.EnvironmentID, err), fault.Public("Failed to read from database. Please try again."), ) } - if len(runtimeSettings.AppRuntimeSetting.RegionConfig) > 0 { - for region, count := range runtimeSettings.AppRuntimeSetting.RegionConfig { - regionConfig[region] = count - } - } - if len(regionConfig) == 0 { + if len(regionalSettings) == 0 { return nil, fault.Wrap( restate.TerminalError(fmt.Errorf("no regions configured for app %s in environment %s", deployment.AppID, deployment.EnvironmentID), 400), fault.Public("No regions configured. Please configure at least one region before deploying."), ) } - regions := make([]string, 0, len(regionConfig)) - for r := range regionConfig { - regions = append(regions, r) - } + topologies := make([]db.InsertDeploymentTopologyParams, 0, len(regionalSettings)) - topologies := make([]db.InsertDeploymentTopologyParams, 0, len(regions)) - - for _, region := range regions { - versionResp, err := hydrav1.NewVersioningServiceClient(ctx, region).NextVersion().Request(&hydrav1.NextVersionRequest{}) + for _, rs := range regionalSettings { + versionResp, err := hydrav1.NewVersioningServiceClient(ctx, rs.RegionName).NextVersion().Request(&hydrav1.NextVersionRequest{}) if err != nil { return nil, fault.Wrap( fmt.Errorf("failed to get next version: %w", err), @@ -431,15 +419,13 @@ func (w *Workflow) createTopologies( ) } - replicas := int32(1) - if count, ok := regionConfig[region]; ok { - replicas = int32(count) - } + replicas := rs.Replicas topologies = append(topologies, db.InsertDeploymentTopologyParams{ WorkspaceID: workspace.ID, DeploymentID: deployment.ID, - Region: region, + Region: rs.RegionName, + RegionID: rs.RegionID, DesiredReplicas: replicas, DesiredStatus: db.DeploymentTopologyDesiredStatusStarting, Version: versionResp.GetVersion(), diff --git a/web/internal/db/src/schema/deployment_topology.ts b/web/internal/db/src/schema/deployment_topology.ts index 37ab0382d56..919dfe9293c 100644 --- a/web/internal/db/src/schema/deployment_topology.ts +++ b/web/internal/db/src/schema/deployment_topology.ts @@ -20,6 +20,7 @@ export const deploymentTopology = mysqlTable( deploymentId: varchar("deployment_id", { length: 64 }).notNull(), region: varchar("region", { length: 64 }).notNull(), + regionId: varchar("region_id", { length: 64 }).notNull().default(""), desiredReplicas: int("desired_replicas").notNull(), From c15b766db4208f8f9c2a7bba4481aa3fd2588eeb Mon Sep 17 00:00:00 2001 From: chronark Date: Sat, 7 Mar 2026 18:45:03 +0100 Subject: [PATCH 17/24] fix: display region names in UI, add local region support - Join regions table when fetching regional settings to get region name - Use region name (not ID) for display in runtime settings - Add "local" flag code to sentinel node types and REGION_INFO --- dev/config/krane.toml | 3 ++- dev/k8s/manifests/krane.yaml | 3 ++- .../network/unkey-flow/components/nodes/types.ts | 3 ++- .../projects/[projectId]/components/region-flag.tsx | 10 ++++++---- .../lib/collections/deploy/environment-settings.ts | 6 +++++- .../trpc/routers/deploy/environment-settings/get.ts | 3 +++ .../dashboard/lib/trpc/routers/deploy/network/utils.ts | 3 ++- web/internal/db/src/schema/app_regional_settings.ts | 9 ++++++++- 8 files changed, 30 insertions(+), 10 deletions(-) diff --git a/dev/config/krane.toml b/dev/config/krane.toml index 2e06a97eba7..68d4fef5e85 100644 --- a/dev/config/krane.toml +++ b/dev/config/krane.toml @@ -1,4 +1,5 @@ -region = "local.dev" +region = "local" +platform = "dev" [control_plane] url = "http://ctrl-api:7091" diff --git a/dev/k8s/manifests/krane.yaml b/dev/k8s/manifests/krane.yaml index 9436204cf13..a67cd40fb52 100644 --- a/dev/k8s/manifests/krane.yaml +++ b/dev/k8s/manifests/krane.yaml @@ -8,7 +8,8 @@ metadata: component: krane data: unkey.toml: | - region = "local.dev" + region = "local" + platform = "dev" [control] url = "http://ctrl-api:7091" diff --git a/web/apps/dashboard/app/(app)/[workspaceSlug]/projects/[projectId]/(overview)/deployments/[deploymentId]/network/unkey-flow/components/nodes/types.ts b/web/apps/dashboard/app/(app)/[workspaceSlug]/projects/[projectId]/(overview)/deployments/[deploymentId]/network/unkey-flow/components/nodes/types.ts index 19cf1a2cec1..ec5c0be6c0b 100644 --- a/web/apps/dashboard/app/(app)/[workspaceSlug]/projects/[projectId]/(overview)/deployments/[deploymentId]/network/unkey-flow/components/nodes/types.ts +++ b/web/apps/dashboard/app/(app)/[workspaceSlug]/projects/[projectId]/(overview)/deployments/[deploymentId]/network/unkey-flow/components/nodes/types.ts @@ -25,7 +25,7 @@ type OriginNode = BaseNode & { type SentinelNode = BaseNode & { metadata: { type: "sentinel"; - flagCode: "us" | "de" | "au" | "jp" | "in" | "br"; + flagCode: "us" | "de" | "au" | "jp" | "in" | "br" | "local"; instances: number; replicas: number; } & BaseMetrics; @@ -77,6 +77,7 @@ const REGION_INFO: Record = { jp: { name: "AP Northeast", location: "Tokyo" }, in: { name: "AP South", location: "Mumbai" }, br: { name: "SA East", location: "São Paulo" }, + local: { name: "Local", location: "Local" }, } as const; const DEFAULT_NODE_WIDTH = 230; diff --git a/web/apps/dashboard/app/(app)/[workspaceSlug]/projects/[projectId]/components/region-flag.tsx b/web/apps/dashboard/app/(app)/[workspaceSlug]/projects/[projectId]/components/region-flag.tsx index cca7e8193fe..a8642c10d76 100644 --- a/web/apps/dashboard/app/(app)/[workspaceSlug]/projects/[projectId]/components/region-flag.tsx +++ b/web/apps/dashboard/app/(app)/[workspaceSlug]/projects/[projectId]/components/region-flag.tsx @@ -1,9 +1,8 @@ import type { FlagCode } from "@/lib/trpc/routers/deploy/network/utils"; import { cn } from "@/lib/utils"; - +import BoringAvatar from "boring-avatars"; type RegionFlagSize = "xs" | "sm" | "md" | "lg"; type RegionFlagShape = "rounded" | "circle"; - type RegionFlagProps = { flagCode: FlagCode; size?: RegionFlagSize; @@ -47,7 +46,6 @@ export function RegionFlag({ }: RegionFlagProps) { const config = sizeConfig[size]; const hasExplicitPadding = config.padding !== ""; - return (
- {flagCode} + {flagCode === "local" ? ( + + ) : ( + {flagCode} + )}
); } diff --git a/web/apps/dashboard/lib/collections/deploy/environment-settings.ts b/web/apps/dashboard/lib/collections/deploy/environment-settings.ts index ea2f348bb9c..9bb28312cca 100644 --- a/web/apps/dashboard/lib/collections/deploy/environment-settings.ts +++ b/web/apps/dashboard/lib/collections/deploy/environment-settings.ts @@ -126,7 +126,11 @@ function flattenSettingsResponse( memoryMib: runtime?.memoryMib ?? 256, command: runtime?.command ?? [], healthcheck: runtime?.healthcheck ?? null, - regions: regional.map((r) => ({ id: r.regionId, name: r.regionId, replicas: r.replicas })), + regions: regional.map((r) => ({ + id: r.region.id, + name: r.region.name, + replicas: r.replicas, + })), shutdownSignal: "SIGTERM", sentinelConfig: runtime?.sentinelConfig, }; diff --git a/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/get.ts b/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/get.ts index c8b24292c71..bddb3bac1a8 100644 --- a/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/get.ts +++ b/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/get.ts @@ -25,6 +25,9 @@ export const getEnvironmentSettings = workspaceProcedure eq(appRegionalSettings.workspaceId, ctx.workspace.id), eq(appRegionalSettings.environmentId, input.environmentId), ), + with: { + region: true, + }, }), ]); diff --git a/web/apps/dashboard/lib/trpc/routers/deploy/network/utils.ts b/web/apps/dashboard/lib/trpc/routers/deploy/network/utils.ts index 2e1a8f16a9e..574d5d9a21d 100644 --- a/web/apps/dashboard/lib/trpc/routers/deploy/network/utils.ts +++ b/web/apps/dashboard/lib/trpc/routers/deploy/network/utils.ts @@ -1,7 +1,7 @@ import type { HealthStatus } from "@/app/(app)/[workspaceSlug]/projects/[projectId]/(overview)/deployments/[deploymentId]/network/unkey-flow/components/nodes/types"; import type { Instance, Sentinel } from "@/lib/db"; -export const flagCodes = ["us", "de", "au", "jp", "in", "br"] as const; +export const flagCodes = ["us", "de", "au", "jp", "in", "br", "local"] as const; export type FlagCode = (typeof flagCodes)[number]; export function mapInstanceStatusToHealth(status: Instance["status"]): HealthStatus { @@ -50,6 +50,7 @@ export function mapRegionToFlag(region: string): FlagCode { ["us-", "us"], ["eu-", "de"], ["sa-", "br"], + ["local", "local"], ] as const; return (prefixMap.find(([prefix]) => region.startsWith(prefix))?.[1] as FlagCode) ?? "us"; diff --git a/web/internal/db/src/schema/app_regional_settings.ts b/web/internal/db/src/schema/app_regional_settings.ts index e7b12d38e19..5d89ab83e9b 100644 --- a/web/internal/db/src/schema/app_regional_settings.ts +++ b/web/internal/db/src/schema/app_regional_settings.ts @@ -2,6 +2,7 @@ import { relations } from "drizzle-orm"; import { bigint, index, int, mysqlTable, uniqueIndex, varchar } from "drizzle-orm/mysql-core"; import { apps } from "./apps"; import { environments } from "./environments"; +import { regions } from "./regions"; import { lifecycleDates } from "./util/lifecycle_dates"; import { workspaces } from "./workspaces"; @@ -24,7 +25,9 @@ export const appRegionalSettings = mysqlTable( replicas: int("replicas").notNull().default(1), // Optional reference to a horizontal autoscaling policy. null = no autoscaling. - horizontalAutoscalingPolicyId: varchar("horizontal_autoscaling_policy_id", { length: 64 }), + horizontalAutoscalingPolicyId: varchar("horizontal_autoscaling_policy_id", { + length: 64, + }), ...lifecycleDates, }, @@ -47,4 +50,8 @@ export const appRegionalSettingsRelations = relations(appRegionalSettings, ({ on fields: [appRegionalSettings.environmentId], references: [environments.id], }), + region: one(regions, { + fields: [appRegionalSettings.regionId], + references: [regions.id], + }), })); From cd898dac6ba4c11b3ecfc8ef7d2f0fcd9de2ed78 Mon Sep 17 00:00:00 2001 From: chronark Date: Sat, 7 Mar 2026 20:14:03 +0100 Subject: [PATCH 18/24] feat: add X-Krane-Platform header, app_id to ApplySentinel, and appId labels - Add X-Krane-Platform header alongside X-Krane-Region in the krane interceptor so ctrl can identify regions by (name, platform) pair - All ctrl RPC handlers now validate both region and platform from headers - Add app_id field to ApplySentinel proto message - Add AppID label to all sentinel sub-resources (Service, PDB, gossip Service, CiliumNetworkPolicy) for consistency with deployment labels - Remove hardcoded region list from sentinel config validation (regions are now dynamic from DB) - Add platform field to sentinel config --- gen/proto/ctrl/v1/cluster.pb.go | 39 ++++++++++++------- svc/ctrl/proto/ctrl/v1/cluster.proto | 15 ++++--- ...get_desired_cilium_network_policy_state.go | 10 +++-- .../rpc_get_desired_deployment_state.go | 10 +++-- .../cluster/rpc_get_desired_sentinel_state.go | 10 +++-- .../cluster/rpc_report_deployment_status.go | 11 +++--- .../cluster/rpc_report_sentinel_status.go | 11 +++--- .../rpc_watch_cilium_network_policies.go | 8 +++- .../services/cluster/rpc_watch_deployments.go | 8 +++- .../services/cluster/rpc_watch_sentinels.go | 8 +++- svc/krane/internal/sentinel/apply.go | 6 +++ svc/krane/internal/sentinel/controller.go | 3 ++ svc/krane/pkg/controlplane/client.go | 6 ++- svc/krane/pkg/controlplane/interceptor.go | 14 ++++--- svc/krane/run.go | 2 + svc/sentinel/config.go | 22 +++-------- svc/sentinel/run.go | 1 + .../dashboard/gen/proto/ctrl/v1/cluster_pb.ts | 21 ++++++---- 18 files changed, 130 insertions(+), 75 deletions(-) diff --git a/gen/proto/ctrl/v1/cluster.pb.go b/gen/proto/ctrl/v1/cluster.pb.go index b11fb6b6e0c..c1bd6254b3b 100644 --- a/gen/proto/ctrl/v1/cluster.pb.go +++ b/gen/proto/ctrl/v1/cluster.pb.go @@ -1084,14 +1084,16 @@ type ApplySentinel struct { WorkspaceId string `protobuf:"bytes,2,opt,name=workspace_id,json=workspaceId,proto3" json:"workspace_id,omitempty"` // project_id identifies the project within the workspace. ProjectId string `protobuf:"bytes,3,opt,name=project_id,json=projectId,proto3" json:"project_id,omitempty"` + // app_id identifies the app within the project. + AppId string `protobuf:"bytes,4,opt,name=app_id,json=appId,proto3" json:"app_id,omitempty"` // environment_id in which the sentinel should exist. - EnvironmentId string `protobuf:"bytes,4,opt,name=environment_id,json=environmentId,proto3" json:"environment_id,omitempty"` + EnvironmentId string `protobuf:"bytes,5,opt,name=environment_id,json=environmentId,proto3" json:"environment_id,omitempty"` // sentinel_id is the unique identifier for this sentinel globally - SentinelId string `protobuf:"bytes,5,opt,name=sentinel_id,json=sentinelId,proto3" json:"sentinel_id,omitempty"` - Image string `protobuf:"bytes,6,opt,name=image,proto3" json:"image,omitempty"` - Replicas int32 `protobuf:"varint,7,opt,name=replicas,proto3" json:"replicas,omitempty"` - CpuMillicores int64 `protobuf:"varint,8,opt,name=cpu_millicores,json=cpuMillicores,proto3" json:"cpu_millicores,omitempty"` - MemoryMib int64 `protobuf:"varint,9,opt,name=memory_mib,json=memoryMib,proto3" json:"memory_mib,omitempty"` + SentinelId string `protobuf:"bytes,6,opt,name=sentinel_id,json=sentinelId,proto3" json:"sentinel_id,omitempty"` + Image string `protobuf:"bytes,7,opt,name=image,proto3" json:"image,omitempty"` + Replicas int32 `protobuf:"varint,8,opt,name=replicas,proto3" json:"replicas,omitempty"` + CpuMillicores int64 `protobuf:"varint,9,opt,name=cpu_millicores,json=cpuMillicores,proto3" json:"cpu_millicores,omitempty"` + MemoryMib int64 `protobuf:"varint,10,opt,name=memory_mib,json=memoryMib,proto3" json:"memory_mib,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1147,6 +1149,13 @@ func (x *ApplySentinel) GetProjectId() string { return "" } +func (x *ApplySentinel) GetAppId() string { + if x != nil { + return x.AppId + } + return "" +} + func (x *ApplySentinel) GetEnvironmentId() string { if x != nil { return x.EnvironmentId @@ -1834,20 +1843,22 @@ const file_ctrl_v1_cluster_proto_rawDesc = "" + "\aversion\x18\x03 \x01(\x04R\aversion\x120\n" + "\x05apply\x18\x01 \x01(\v2\x18.ctrl.v1.ApplyDeploymentH\x00R\x05apply\x123\n" + "\x06delete\x18\x02 \x01(\v2\x19.ctrl.v1.DeleteDeploymentH\x00R\x06deleteB\a\n" + - "\x05state\"\xac\x02\n" + + "\x05state\"\xc3\x02\n" + "\rApplySentinel\x12\x19\n" + "\bk8s_name\x18\x01 \x01(\tR\ak8sName\x12!\n" + "\fworkspace_id\x18\x02 \x01(\tR\vworkspaceId\x12\x1d\n" + "\n" + - "project_id\x18\x03 \x01(\tR\tprojectId\x12%\n" + - "\x0eenvironment_id\x18\x04 \x01(\tR\renvironmentId\x12\x1f\n" + - "\vsentinel_id\x18\x05 \x01(\tR\n" + + "project_id\x18\x03 \x01(\tR\tprojectId\x12\x15\n" + + "\x06app_id\x18\x04 \x01(\tR\x05appId\x12%\n" + + "\x0eenvironment_id\x18\x05 \x01(\tR\renvironmentId\x12\x1f\n" + + "\vsentinel_id\x18\x06 \x01(\tR\n" + "sentinelId\x12\x14\n" + - "\x05image\x18\x06 \x01(\tR\x05image\x12\x1a\n" + - "\breplicas\x18\a \x01(\x05R\breplicas\x12%\n" + - "\x0ecpu_millicores\x18\b \x01(\x03R\rcpuMillicores\x12\x1d\n" + + "\x05image\x18\a \x01(\tR\x05image\x12\x1a\n" + + "\breplicas\x18\b \x01(\x05R\breplicas\x12%\n" + + "\x0ecpu_millicores\x18\t \x01(\x03R\rcpuMillicores\x12\x1d\n" + "\n" + - "memory_mib\x18\t \x01(\x03R\tmemoryMib\"+\n" + + "memory_mib\x18\n" + + " \x01(\x03R\tmemoryMib\"+\n" + "\x0eDeleteSentinel\x12\x19\n" + "\bk8s_name\x18\x01 \x01(\tR\ak8sName\"\xf1\x04\n" + "\x0fApplyDeployment\x12#\n" + diff --git a/svc/ctrl/proto/ctrl/v1/cluster.proto b/svc/ctrl/proto/ctrl/v1/cluster.proto index c5fc7c13dd3..a4b780e1823 100644 --- a/svc/ctrl/proto/ctrl/v1/cluster.proto +++ b/svc/ctrl/proto/ctrl/v1/cluster.proto @@ -232,16 +232,19 @@ message ApplySentinel { // project_id identifies the project within the workspace. string project_id = 3; + // app_id identifies the app within the project. + string app_id = 4; + // environment_id in which the sentinel should exist. - string environment_id = 4; + string environment_id = 5; // sentinel_id is the unique identifier for this sentinel globally - string sentinel_id = 5; + string sentinel_id = 6; - string image = 6; - int32 replicas = 7; - int64 cpu_millicores = 8; - int64 memory_mib = 9; + string image = 7; + int32 replicas = 8; + int64 cpu_millicores = 9; + int64 memory_mib = 10; } // DeleteSentinel identifies a sentinel to remove from the cluster. diff --git a/svc/ctrl/services/cluster/rpc_get_desired_cilium_network_policy_state.go b/svc/ctrl/services/cluster/rpc_get_desired_cilium_network_policy_state.go index 19af6b56e9b..9b69486617c 100644 --- a/svc/ctrl/services/cluster/rpc_get_desired_cilium_network_policy_state.go +++ b/svc/ctrl/services/cluster/rpc_get_desired_cilium_network_policy_state.go @@ -13,8 +13,8 @@ import ( // in the caller's region. This is a point query alternative to [Service.WatchCiliumNetworkPolicies] // for cases where an agent needs to fetch state for a specific policy rather than streaming all changes. // -// Returns CodeUnauthenticated if bearer token is invalid, CodeInvalidArgument if the -// X-Krane-Region header is missing, CodeNotFound if no policy exists with the given ID +// Returns CodeUnauthenticated if bearer token is invalid, CodeInvalidArgument if +// region or platform is missing, CodeNotFound if no policy exists with the given ID // in the specified region, or CodeInternal for database errors. func (s *Service) GetDesiredCiliumNetworkPolicyState( ctx context.Context, @@ -25,7 +25,11 @@ func (s *Service) GetDesiredCiliumNetworkPolicyState( } region := req.Header().Get("X-Krane-Region") - if err := assert.NotEmpty(region, "region is required"); err != nil { + platform := req.Header().Get("X-Krane-Platform") + if err := assert.All( + assert.NotEmpty(region, "region is required"), + assert.NotEmpty(platform, "platform is required"), + ); err != nil { return nil, connect.NewError(connect.CodeInvalidArgument, err) } diff --git a/svc/ctrl/services/cluster/rpc_get_desired_deployment_state.go b/svc/ctrl/services/cluster/rpc_get_desired_deployment_state.go index 5dc78b594d3..0eec1352e5f 100644 --- a/svc/ctrl/services/cluster/rpc_get_desired_deployment_state.go +++ b/svc/ctrl/services/cluster/rpc_get_desired_deployment_state.go @@ -20,8 +20,8 @@ import ( // (for archived or standby states) based on the deployment_topology's desired_state in the // database. Unhandled desired states result in CodeInternal. // -// Returns CodeUnauthenticated if bearer token is invalid, CodeInvalidArgument if the -// X-Krane-Region header is missing, CodeNotFound if no deployment exists with the given +// Returns CodeUnauthenticated if bearer token is invalid, CodeInvalidArgument if +// region or platform is missing, CodeNotFound if no deployment exists with the given // ID in the specified region, or CodeInternal for database errors or unhandled states. func (s *Service) GetDesiredDeploymentState(ctx context.Context, req *connect.Request[ctrlv1.GetDesiredDeploymentStateRequest]) (*connect.Response[ctrlv1.DeploymentState], error) { @@ -30,7 +30,11 @@ func (s *Service) GetDesiredDeploymentState(ctx context.Context, req *connect.Re } region := req.Header().Get("X-Krane-Region") - if err := assert.NotEmpty(region, "region is required"); err != nil { + platform := req.Header().Get("X-Krane-Platform") + if err := assert.All( + assert.NotEmpty(region, "region is required"), + assert.NotEmpty(platform, "platform is required"), + ); err != nil { return nil, connect.NewError(connect.CodeInvalidArgument, err) } diff --git a/svc/ctrl/services/cluster/rpc_get_desired_sentinel_state.go b/svc/ctrl/services/cluster/rpc_get_desired_sentinel_state.go index 71647378ba4..4660b911962 100644 --- a/svc/ctrl/services/cluster/rpc_get_desired_sentinel_state.go +++ b/svc/ctrl/services/cluster/rpc_get_desired_sentinel_state.go @@ -19,8 +19,8 @@ import ( // (for archived or standby states) based on the sentinel's desired_state in the database. // Unhandled desired states result in CodeInternal. // -// Returns CodeUnauthenticated if bearer token is invalid, CodeInvalidArgument if the -// X-Krane-Region header is missing, CodeNotFound if no sentinel exists with the given ID, +// Returns CodeUnauthenticated if bearer token is invalid, CodeInvalidArgument if +// region or platform is missing, CodeNotFound if no sentinel exists with the given ID, // or CodeInternal for database errors or unhandled states. func (s *Service) GetDesiredSentinelState(ctx context.Context, req *connect.Request[ctrlv1.GetDesiredSentinelStateRequest]) (*connect.Response[ctrlv1.SentinelState], error) { @@ -29,7 +29,11 @@ func (s *Service) GetDesiredSentinelState(ctx context.Context, req *connect.Requ } region := req.Header().Get("X-Krane-Region") - if err := assert.NotEmpty(region, "region is required"); err != nil { + platform := req.Header().Get("X-Krane-Platform") + if err := assert.All( + assert.NotEmpty(region, "region is required"), + assert.NotEmpty(platform, "platform is required"), + ); err != nil { return nil, connect.NewError(connect.CodeInvalidArgument, err) } diff --git a/svc/ctrl/services/cluster/rpc_report_deployment_status.go b/svc/ctrl/services/cluster/rpc_report_deployment_status.go index 07e07b1180d..19727811b90 100644 --- a/svc/ctrl/services/cluster/rpc_report_deployment_status.go +++ b/svc/ctrl/services/cluster/rpc_report_deployment_status.go @@ -33,15 +33,16 @@ func (s *Service) ReportDeploymentStatus(ctx context.Context, req *connect.Reque return nil, err } region := req.Header().Get("X-Krane-Region") + platform := req.Header().Get("X-Krane-Platform") - err := assert.All( + if err := assert.All( assert.NotEmpty(region, "region is required"), - ) - if err != nil { - return nil, err + assert.NotEmpty(platform, "platform is required"), + ); err != nil { + return nil, connect.NewError(connect.CodeInvalidArgument, err) } - err = db.TxRetry(ctx, s.db.RW(), func(txCtx context.Context, tx db.DBTX) error { + err := db.TxRetry(ctx, s.db.RW(), func(txCtx context.Context, tx db.DBTX) error { switch msg := req.Msg.GetChange().(type) { case *ctrlv1.ReportDeploymentStatusRequest_Update_: diff --git a/svc/ctrl/services/cluster/rpc_report_sentinel_status.go b/svc/ctrl/services/cluster/rpc_report_sentinel_status.go index b258c47740c..ef0ffeaa457 100644 --- a/svc/ctrl/services/cluster/rpc_report_sentinel_status.go +++ b/svc/ctrl/services/cluster/rpc_report_sentinel_status.go @@ -29,12 +29,13 @@ func (s *Service) ReportSentinelStatus(ctx context.Context, req *connect.Request } region := req.Header().Get("X-Krane-Region") + platform := req.Header().Get("X-Krane-Platform") - err := assert.All( + if err := assert.All( assert.NotEmpty(region, "region is required"), - ) - if err != nil { - return nil, err + assert.NotEmpty(platform, "platform is required"), + ); err != nil { + return nil, connect.NewError(connect.CodeInvalidArgument, err) } var health db.SentinelsHealth @@ -48,7 +49,7 @@ func (s *Service) ReportSentinelStatus(ctx context.Context, req *connect.Request case ctrlv1.Health_HEALTH_UNSPECIFIED: health = db.SentinelsHealthUnknown } - err = db.Query.UpdateSentinelAvailableReplicasAndHealth(ctx, s.db.RW(), db.UpdateSentinelAvailableReplicasAndHealthParams{ + err := db.Query.UpdateSentinelAvailableReplicasAndHealth(ctx, s.db.RW(), db.UpdateSentinelAvailableReplicasAndHealthParams{ K8sName: req.Msg.GetK8SName(), AvailableReplicas: req.Msg.GetAvailableReplicas(), Health: health, diff --git a/svc/ctrl/services/cluster/rpc_watch_cilium_network_policies.go b/svc/ctrl/services/cluster/rpc_watch_cilium_network_policies.go index 91f0c705d92..c7366a90aa9 100644 --- a/svc/ctrl/services/cluster/rpc_watch_cilium_network_policies.go +++ b/svc/ctrl/services/cluster/rpc_watch_cilium_network_policies.go @@ -34,8 +34,12 @@ func (s *Service) WatchCiliumNetworkPolicies( return err } - region := req.Msg.GetRegion() - if err := assert.NotEmpty(region, "region is required"); err != nil { + region := req.Header().Get("X-Krane-Region") + platform := req.Header().Get("X-Krane-Platform") + if err := assert.All( + assert.NotEmpty(region, "region is required"), + assert.NotEmpty(platform, "platform is required"), + ); err != nil { return connect.NewError(connect.CodeInvalidArgument, err) } diff --git a/svc/ctrl/services/cluster/rpc_watch_deployments.go b/svc/ctrl/services/cluster/rpc_watch_deployments.go index 585c9965774..5c59007f948 100644 --- a/svc/ctrl/services/cluster/rpc_watch_deployments.go +++ b/svc/ctrl/services/cluster/rpc_watch_deployments.go @@ -37,8 +37,12 @@ func (s *Service) WatchDeployments( return err } - region := req.Msg.GetRegion() - if err := assert.NotEmpty(region, "region is required"); err != nil { + region := req.Header().Get("X-Krane-Region") + platform := req.Header().Get("X-Krane-Platform") + if err := assert.All( + assert.NotEmpty(region, "region is required"), + assert.NotEmpty(platform, "platform is required"), + ); err != nil { return connect.NewError(connect.CodeInvalidArgument, err) } diff --git a/svc/ctrl/services/cluster/rpc_watch_sentinels.go b/svc/ctrl/services/cluster/rpc_watch_sentinels.go index 53178254d52..0f79a678f86 100644 --- a/svc/ctrl/services/cluster/rpc_watch_sentinels.go +++ b/svc/ctrl/services/cluster/rpc_watch_sentinels.go @@ -34,8 +34,12 @@ func (s *Service) WatchSentinels( return err } - region := req.Msg.GetRegion() - if err := assert.NotEmpty(region, "region is required"); err != nil { + region := req.Header().Get("X-Krane-Region") + platform := req.Header().Get("X-Krane-Platform") + if err := assert.All( + assert.NotEmpty(region, "region is required"), + assert.NotEmpty(platform, "platform is required"), + ); err != nil { return connect.NewError(connect.CodeInvalidArgument, err) } versionCursor := req.Msg.GetVersionLastSeen() diff --git a/svc/krane/internal/sentinel/apply.go b/svc/krane/internal/sentinel/apply.go index bd8b081658c..11ca456b39b 100644 --- a/svc/krane/internal/sentinel/apply.go +++ b/svc/krane/internal/sentinel/apply.go @@ -130,6 +130,7 @@ func (c *Controller) ensureSentinelExists(ctx context.Context, sentinel *ctrlv1. SentinelID: sentinel.GetSentinelId(), WorkspaceID: sentinel.GetWorkspaceId(), EnvironmentID: sentinel.GetEnvironmentId(), + Platform: c.platform, Region: c.region, HttpPort: SentinelPort, Database: config.DatabaseConfig{ @@ -169,6 +170,7 @@ func (c *Controller) ensureSentinelExists(ctx context.Context, sentinel *ctrlv1. Labels: labels.New(). WorkspaceID(sentinel.GetWorkspaceId()). ProjectID(sentinel.GetProjectId()). + AppID(sentinel.GetAppId()). EnvironmentID(sentinel.GetEnvironmentId()). SentinelID(sentinel.GetSentinelId()). ComponentSentinel(). @@ -319,6 +321,7 @@ func (c *Controller) ensureServiceExists(ctx context.Context, sentinel *ctrlv1.A Labels: labels.New(). WorkspaceID(sentinel.GetWorkspaceId()). ProjectID(sentinel.GetProjectId()). + AppID(sentinel.GetAppId()). EnvironmentID(sentinel.GetEnvironmentId()). SentinelID(sentinel.GetSentinelId()). ManagedByKrane(). @@ -388,6 +391,7 @@ func (c *Controller) ensurePDBExists(ctx context.Context, sentinel *ctrlv1.Apply Labels: labels.New(). WorkspaceID(sentinel.GetWorkspaceId()). ProjectID(sentinel.GetProjectId()). + AppID(sentinel.GetAppId()). EnvironmentID(sentinel.GetEnvironmentId()). SentinelID(sentinel.GetSentinelId()). ManagedByKrane(). @@ -440,6 +444,7 @@ func (c *Controller) ensureGossipServiceExists(ctx context.Context, sentinel *ct Labels: labels.New(). WorkspaceID(sentinel.GetWorkspaceId()). ProjectID(sentinel.GetProjectId()). + AppID(sentinel.GetAppId()). EnvironmentID(sentinel.GetEnvironmentId()). SentinelID(sentinel.GetSentinelId()). ComponentGossipLAN(), @@ -495,6 +500,7 @@ func (c *Controller) ensureGossipCiliumPolicyExists(ctx context.Context, sentine "labels": labels.New(). WorkspaceID(sentinel.GetWorkspaceId()). ProjectID(sentinel.GetProjectId()). + AppID(sentinel.GetAppId()). EnvironmentID(sentinel.GetEnvironmentId()). SentinelID(sentinel.GetSentinelId()). ComponentGossipLAN(), diff --git a/svc/krane/internal/sentinel/controller.go b/svc/krane/internal/sentinel/controller.go index 93b126ad6ce..ab93f279985 100644 --- a/svc/krane/internal/sentinel/controller.go +++ b/svc/krane/internal/sentinel/controller.go @@ -26,6 +26,7 @@ type Controller struct { done chan struct{} stopOnce sync.Once region string + platform string versionLastSeen uint64 } @@ -33,6 +34,7 @@ type Controller struct { type Config struct { Cluster ctrl.ClusterServiceClient Region string + Platform string ClientSet kubernetes.Interface DynamicClient dynamic.Interface } @@ -46,6 +48,7 @@ func New(cfg Config) *Controller { cb: circuitbreaker.New[any]("sentinel_state_update"), done: make(chan struct{}), region: cfg.Region, + platform: cfg.Platform, versionLastSeen: 0, stopOnce: sync.Once{}, } diff --git a/svc/krane/pkg/controlplane/client.go b/svc/krane/pkg/controlplane/client.go index f97b5458b1e..77b7cc0152f 100644 --- a/svc/krane/pkg/controlplane/client.go +++ b/svc/krane/pkg/controlplane/client.go @@ -28,6 +28,10 @@ type ClientConfig struct { // Region identifies the geographical region of this client instance. // This value is added as the X-Krane-Region header for proper request routing. Region string + + // Platform identifies the infrastructure provider (e.g. "aws", "local"). + // This value is added as the X-Krane-Platform header. + Platform string } // NewClient creates a new control plane client with the specified configuration. @@ -70,6 +74,6 @@ func NewClient(cfg ClientConfig) ctrl.ClusterServiceClient { Transport: transport, }, cfg.URL, - connect.WithInterceptors(connectInterceptor(cfg.Region, cfg.BearerToken)), + connect.WithInterceptors(connectInterceptor(cfg.Region, cfg.Platform, cfg.BearerToken)), )) } diff --git a/svc/krane/pkg/controlplane/interceptor.go b/svc/krane/pkg/controlplane/interceptor.go index 9a0738a4715..6cc9ffb9384 100644 --- a/svc/krane/pkg/controlplane/interceptor.go +++ b/svc/krane/pkg/controlplane/interceptor.go @@ -16,10 +16,11 @@ import ( // - X-Krane-Region header for routing // // This is the recommended way to create interceptors for control plane clients. -func connectInterceptor(region, bearer string) connect.Interceptor { +func connectInterceptor(region, platform, bearer string) connect.Interceptor { return &authInterceptor{ - region: region, - bearer: bearer, + region: region, + platform: platform, + bearer: bearer, } } @@ -28,8 +29,9 @@ func connectInterceptor(region, bearer string) connect.Interceptor { // It automatically adds authentication and routing metadata to all outgoing requests. // The interceptor is stateless and safe for concurrent use. type authInterceptor struct { - region string - bearer string + region string + platform string + bearer string } // WrapUnary intercepts unary RPC calls by adding required headers before forwarding @@ -122,10 +124,12 @@ func (s *streamingClientInterceptor) RequestHeader() http.Header { // // This method injects: // - X-Krane-Region: The client's geographical region +// - X-Krane-Platform: The infrastructure provider (e.g. "aws", "local") // - Authorization: Bearer token for authentication // // All headers use Set() to overwrite any existing values. func (i *authInterceptor) setHeaders(header http.Header) { header.Set("X-Krane-Region", i.region) + header.Set("X-Krane-Platform", i.platform) header.Set("Authorization", fmt.Sprintf("Bearer %s", i.bearer)) } diff --git a/svc/krane/run.go b/svc/krane/run.go index 6c13ad618f6..e44c8ec8175 100644 --- a/svc/krane/run.go +++ b/svc/krane/run.go @@ -96,6 +96,7 @@ func Run(ctx context.Context, cfg Config) error { URL: cfg.Control.URL, BearerToken: cfg.Control.Token, Region: cfg.Region, + Platform: cfg.Platform, }) inClusterConfig, err := rest.InClusterConfig() @@ -143,6 +144,7 @@ func Run(ctx context.Context, cfg Config) error { DynamicClient: dynamicClient, Cluster: cluster, Region: cfg.Region, + Platform: cfg.Platform, }) if err := sentinelCtrl.Start(ctx); err != nil { return fmt.Errorf("failed to start sentinel controller: %w", err) diff --git a/svc/sentinel/config.go b/svc/sentinel/config.go index 1eb0f053c0a..7ee81b189ac 100644 --- a/svc/sentinel/config.go +++ b/svc/sentinel/config.go @@ -1,9 +1,6 @@ package sentinel import ( - "fmt" - "slices" - "github.com/unkeyed/unkey/pkg/config" ) @@ -46,9 +43,12 @@ type Config struct { // handles all of them based on the deployment ID passed in each request. EnvironmentID string `toml:"environment_id" config:"required,nonempty"` - // Region is the geographic region identifier (e.g. "us-east-1.aws"). + // Platform identifies the underlying cloud platform this sentinel is running on. + Platform string `toml:"platform" config:"required,nonempty,oneof=aws,local"` + + // Region is the geographic region identifier (e.g. "us-east-1"). // Included in structured logs and used for routing decisions. - Region string `toml:"region" config:"required,oneof=local.dev|us-east-1.aws|us-east-2.aws|us-west-1.aws|us-west-2.aws|eu-central-1.aws"` + Region string `toml:"region" config:"required,nonempty"` // HttpPort is the TCP port the sentinel server binds to. HttpPort int `toml:"http_port" config:"default=8080,min=1,max=65535"` @@ -75,18 +75,6 @@ type Config struct { // struct tags alone. It implements [config.Validator] so that [config.Load] // calls it automatically after tag-level validation. func (c *Config) Validate() error { - validRegions := []string{ - "local.dev", - "us-east-1.aws", - "us-east-2.aws", - "us-west-1.aws", - "us-west-2.aws", - "eu-central-1.aws", - } - - if !slices.Contains(validRegions, c.Region) { - return fmt.Errorf("invalid region: %s, must be one of %v", c.Region, validRegions) - } return nil } diff --git a/svc/sentinel/run.go b/svc/sentinel/run.go index da030393348..7cec3c50a9c 100644 --- a/svc/sentinel/run.go +++ b/svc/sentinel/run.go @@ -68,6 +68,7 @@ func Run(ctx context.Context, cfg Config) error { slog.String("sentinelID", cfg.SentinelID), slog.String("workspaceID", cfg.WorkspaceID), slog.String("environmentID", cfg.EnvironmentID), + slog.String("platform", cfg.Platform), slog.String("region", cfg.Region), slog.String("version", version.Version), )) diff --git a/web/apps/dashboard/gen/proto/ctrl/v1/cluster_pb.ts b/web/apps/dashboard/gen/proto/ctrl/v1/cluster_pb.ts index d5b9c688a25..cc70282a350 100644 --- a/web/apps/dashboard/gen/proto/ctrl/v1/cluster_pb.ts +++ b/web/apps/dashboard/gen/proto/ctrl/v1/cluster_pb.ts @@ -20,7 +20,7 @@ import type { Message } from "@bufbuild/protobuf"; * Describes the file ctrl/v1/cluster.proto. */ export const file_ctrl_v1_cluster: GenFile = /*@__PURE__*/ - fileDesc("ChVjdHJsL3YxL2NsdXN0ZXIucHJvdG8SB2N0cmwudjEiTgohV2F0Y2hDaWxpdW1OZXR3b3JrUG9saWNpZXNSZXF1ZXN0Eg4KBnJlZ2lvbhgBIAEoCRIZChF2ZXJzaW9uX2xhc3Rfc2VlbhgCIAEoBCJ1ChhBcHBseUNpbGl1bU5ldHdvcmtQb2xpY3kSFQoNazhzX25hbWVzcGFjZRgBIAEoCRIQCghrOHNfbmFtZRgCIAEoCRIgChhjaWxpdW1fbmV0d29ya19wb2xpY3lfaWQYAyABKAkSDgoGcG9saWN5GAQgASgMIkQKGURlbGV0ZUNpbGl1bU5ldHdvcmtQb2xpY3kSFQoNazhzX25hbWVzcGFjZRgBIAEoCRIQCghrOHNfbmFtZRgCIAEoCSKeAQoYQ2lsaXVtTmV0d29ya1BvbGljeVN0YXRlEg8KB3ZlcnNpb24YAyABKAQSMgoFYXBwbHkYASABKAsyIS5jdHJsLnYxLkFwcGx5Q2lsaXVtTmV0d29ya1BvbGljeUgAEjQKBmRlbGV0ZRgCIAEoCzIiLmN0cmwudjEuRGVsZXRlQ2lsaXVtTmV0d29ya1BvbGljeUgAQgcKBXN0YXRlIk0KKUdldERlc2lyZWRDaWxpdW1OZXR3b3JrUG9saWN5U3RhdGVSZXF1ZXN0EiAKGGNpbGl1bV9uZXR3b3JrX3BvbGljeV9pZBgBIAEoCSI1Ch5HZXREZXNpcmVkU2VudGluZWxTdGF0ZVJlcXVlc3QSEwoLc2VudGluZWxfaWQYASABKAkiOQogR2V0RGVzaXJlZERlcGxveW1lbnRTdGF0ZVJlcXVlc3QSFQoNZGVwbG95bWVudF9pZBgBIAEoCSK3BAodUmVwb3J0RGVwbG95bWVudFN0YXR1c1JlcXVlc3QSPwoGdXBkYXRlGAEgASgLMi0uY3RybC52MS5SZXBvcnREZXBsb3ltZW50U3RhdHVzUmVxdWVzdC5VcGRhdGVIABI/CgZkZWxldGUYAiABKAsyLS5jdHJsLnYxLlJlcG9ydERlcGxveW1lbnRTdGF0dXNSZXF1ZXN0LkRlbGV0ZUgAGu0CCgZVcGRhdGUSEAoIazhzX25hbWUYASABKAkSSQoJaW5zdGFuY2VzGAIgAygLMjYuY3RybC52MS5SZXBvcnREZXBsb3ltZW50U3RhdHVzUmVxdWVzdC5VcGRhdGUuSW5zdGFuY2UahQIKCEluc3RhbmNlEhAKCGs4c19uYW1lGAEgASgJEg8KB2FkZHJlc3MYAiABKAkSFgoOY3B1X21pbGxpY29yZXMYAyABKAMSEgoKbWVtb3J5X21pYhgEIAEoAxJNCgZzdGF0dXMYBSABKA4yPS5jdHJsLnYxLlJlcG9ydERlcGxveW1lbnRTdGF0dXNSZXF1ZXN0LlVwZGF0ZS5JbnN0YW5jZS5TdGF0dXMiWwoGU3RhdHVzEhYKElNUQVRVU19VTlNQRUNJRklFRBAAEhIKDlNUQVRVU19QRU5ESU5HEAESEgoOU1RBVFVTX1JVTk5JTkcQAhIRCg1TVEFUVVNfRkFJTEVEEAMaGgoGRGVsZXRlEhAKCGs4c19uYW1lGAEgASgJQggKBmNoYW5nZSIgCh5SZXBvcnREZXBsb3ltZW50U3RhdHVzUmVzcG9uc2UibAobUmVwb3J0U2VudGluZWxTdGF0dXNSZXF1ZXN0EhAKCGs4c19uYW1lGAEgASgJEhoKEmF2YWlsYWJsZV9yZXBsaWNhcxgCIAEoBRIfCgZoZWFsdGgYAyABKA4yDy5jdHJsLnYxLkhlYWx0aCIeChxSZXBvcnRTZW50aW5lbFN0YXR1c1Jlc3BvbnNlIkQKF1dhdGNoRGVwbG95bWVudHNSZXF1ZXN0Eg4KBnJlZ2lvbhgBIAEoCRIZChF2ZXJzaW9uX2xhc3Rfc2VlbhgCIAEoBCJCChVXYXRjaFNlbnRpbmVsc1JlcXVlc3QSDgoGcmVnaW9uGAEgASgJEhkKEXZlcnNpb25fbGFzdF9zZWVuGAIgASgEIn0KDVNlbnRpbmVsU3RhdGUSDwoHdmVyc2lvbhgDIAEoBBInCgVhcHBseRgBIAEoCzIWLmN0cmwudjEuQXBwbHlTZW50aW5lbEgAEikKBmRlbGV0ZRgCIAEoCzIXLmN0cmwudjEuRGVsZXRlU2VudGluZWxIAEIHCgVzdGF0ZSKDAQoPRGVwbG95bWVudFN0YXRlEg8KB3ZlcnNpb24YAyABKAQSKQoFYXBwbHkYASABKAsyGC5jdHJsLnYxLkFwcGx5RGVwbG95bWVudEgAEisKBmRlbGV0ZRgCIAEoCzIZLmN0cmwudjEuRGVsZXRlRGVwbG95bWVudEgAQgcKBXN0YXRlIsUBCg1BcHBseVNlbnRpbmVsEhAKCGs4c19uYW1lGAEgASgJEhQKDHdvcmtzcGFjZV9pZBgCIAEoCRISCgpwcm9qZWN0X2lkGAMgASgJEhYKDmVudmlyb25tZW50X2lkGAQgASgJEhMKC3NlbnRpbmVsX2lkGAUgASgJEg0KBWltYWdlGAYgASgJEhAKCHJlcGxpY2FzGAcgASgFEhYKDmNwdV9taWxsaWNvcmVzGAggASgDEhIKCm1lbW9yeV9taWIYCSABKAMiIgoORGVsZXRlU2VudGluZWwSEAoIazhzX25hbWUYASABKAkinwMKD0FwcGx5RGVwbG95bWVudBIVCg1rOHNfbmFtZXNwYWNlGAEgASgJEhAKCGs4c19uYW1lGAIgASgJEhQKDHdvcmtzcGFjZV9pZBgDIAEoCRISCgpwcm9qZWN0X2lkGAQgASgJEhYKDmVudmlyb25tZW50X2lkGAUgASgJEhUKDWRlcGxveW1lbnRfaWQYBiABKAkSDQoFaW1hZ2UYByABKAkSEAoIcmVwbGljYXMYCCABKAUSFgoOY3B1X21pbGxpY29yZXMYCSABKAMSEgoKbWVtb3J5X21pYhgKIAEoAxIVCghidWlsZF9pZBgLIAEoCUgAiAEBEicKH2VuY3J5cHRlZF9lbnZpcm9ubWVudF92YXJpYWJsZXMYDCABKAwSDwoHY29tbWFuZBgNIAMoCRIMCgRwb3J0GA4gASgFEhcKD3NodXRkb3duX3NpZ25hbBgPIAEoCRIYCgtoZWFsdGhjaGVjaxgRIAEoDEgBiAEBEg4KBmFwcF9pZBgSIAEoCUILCglfYnVpbGRfaWRCDgoMX2hlYWx0aGNoZWNrIjsKEERlbGV0ZURlcGxveW1lbnQSFQoNazhzX25hbWVzcGFjZRgBIAEoCRIQCghrOHNfbmFtZRgCIAEoCSI0ChBIZWFydGJlYXRSZXF1ZXN0Eg4KBnJlZ2lvbhgBIAEoCRIQCghwbGF0Zm9ybRgCIAEoCSITChFIZWFydGJlYXRSZXNwb25zZSpdCgZIZWFsdGgSFgoSSEVBTFRIX1VOU1BFQ0lGSUVEEAASEgoOSEVBTFRIX0hFQUxUSFkQARIUChBIRUFMVEhfVU5IRUFMVEhZEAISEQoNSEVBTFRIX1BBVVNFRBADMuwGCg5DbHVzdGVyU2VydmljZRJQChBXYXRjaERlcGxveW1lbnRzEiAuY3RybC52MS5XYXRjaERlcGxveW1lbnRzUmVxdWVzdBoYLmN0cmwudjEuRGVwbG95bWVudFN0YXRlMAESSgoOV2F0Y2hTZW50aW5lbHMSHi5jdHJsLnYxLldhdGNoU2VudGluZWxzUmVxdWVzdBoWLmN0cmwudjEuU2VudGluZWxTdGF0ZTABEloKF0dldERlc2lyZWRTZW50aW5lbFN0YXRlEicuY3RybC52MS5HZXREZXNpcmVkU2VudGluZWxTdGF0ZVJlcXVlc3QaFi5jdHJsLnYxLlNlbnRpbmVsU3RhdGUSYwoUUmVwb3J0U2VudGluZWxTdGF0dXMSJC5jdHJsLnYxLlJlcG9ydFNlbnRpbmVsU3RhdHVzUmVxdWVzdBolLmN0cmwudjEuUmVwb3J0U2VudGluZWxTdGF0dXNSZXNwb25zZRJgChlHZXREZXNpcmVkRGVwbG95bWVudFN0YXRlEikuY3RybC52MS5HZXREZXNpcmVkRGVwbG95bWVudFN0YXRlUmVxdWVzdBoYLmN0cmwudjEuRGVwbG95bWVudFN0YXRlEmkKFlJlcG9ydERlcGxveW1lbnRTdGF0dXMSJi5jdHJsLnYxLlJlcG9ydERlcGxveW1lbnRTdGF0dXNSZXF1ZXN0GicuY3RybC52MS5SZXBvcnREZXBsb3ltZW50U3RhdHVzUmVzcG9uc2USbQoaV2F0Y2hDaWxpdW1OZXR3b3JrUG9saWNpZXMSKi5jdHJsLnYxLldhdGNoQ2lsaXVtTmV0d29ya1BvbGljaWVzUmVxdWVzdBohLmN0cmwudjEuQ2lsaXVtTmV0d29ya1BvbGljeVN0YXRlMAESewoiR2V0RGVzaXJlZENpbGl1bU5ldHdvcmtQb2xpY3lTdGF0ZRIyLmN0cmwudjEuR2V0RGVzaXJlZENpbGl1bU5ldHdvcmtQb2xpY3lTdGF0ZVJlcXVlc3QaIS5jdHJsLnYxLkNpbGl1bU5ldHdvcmtQb2xpY3lTdGF0ZRJCCglIZWFydGJlYXQSGS5jdHJsLnYxLkhlYXJ0YmVhdFJlcXVlc3QaGi5jdHJsLnYxLkhlYXJ0YmVhdFJlc3BvbnNlQosBCgtjb20uY3RybC52MUIMQ2x1c3RlclByb3RvUAFaMWdpdGh1Yi5jb20vdW5rZXllZC91bmtleS9nZW4vcHJvdG8vY3RybC92MTtjdHJsdjGiAgNDWFiqAgdDdHJsLlYxygIHQ3RybFxWMeICE0N0cmxcVjFcR1BCTWV0YWRhdGHqAghDdHJsOjpWMWIGcHJvdG8z"); + fileDesc("ChVjdHJsL3YxL2NsdXN0ZXIucHJvdG8SB2N0cmwudjEiTgohV2F0Y2hDaWxpdW1OZXR3b3JrUG9saWNpZXNSZXF1ZXN0Eg4KBnJlZ2lvbhgBIAEoCRIZChF2ZXJzaW9uX2xhc3Rfc2VlbhgCIAEoBCJ1ChhBcHBseUNpbGl1bU5ldHdvcmtQb2xpY3kSFQoNazhzX25hbWVzcGFjZRgBIAEoCRIQCghrOHNfbmFtZRgCIAEoCRIgChhjaWxpdW1fbmV0d29ya19wb2xpY3lfaWQYAyABKAkSDgoGcG9saWN5GAQgASgMIkQKGURlbGV0ZUNpbGl1bU5ldHdvcmtQb2xpY3kSFQoNazhzX25hbWVzcGFjZRgBIAEoCRIQCghrOHNfbmFtZRgCIAEoCSKeAQoYQ2lsaXVtTmV0d29ya1BvbGljeVN0YXRlEg8KB3ZlcnNpb24YAyABKAQSMgoFYXBwbHkYASABKAsyIS5jdHJsLnYxLkFwcGx5Q2lsaXVtTmV0d29ya1BvbGljeUgAEjQKBmRlbGV0ZRgCIAEoCzIiLmN0cmwudjEuRGVsZXRlQ2lsaXVtTmV0d29ya1BvbGljeUgAQgcKBXN0YXRlIk0KKUdldERlc2lyZWRDaWxpdW1OZXR3b3JrUG9saWN5U3RhdGVSZXF1ZXN0EiAKGGNpbGl1bV9uZXR3b3JrX3BvbGljeV9pZBgBIAEoCSI1Ch5HZXREZXNpcmVkU2VudGluZWxTdGF0ZVJlcXVlc3QSEwoLc2VudGluZWxfaWQYASABKAkiOQogR2V0RGVzaXJlZERlcGxveW1lbnRTdGF0ZVJlcXVlc3QSFQoNZGVwbG95bWVudF9pZBgBIAEoCSK3BAodUmVwb3J0RGVwbG95bWVudFN0YXR1c1JlcXVlc3QSPwoGdXBkYXRlGAEgASgLMi0uY3RybC52MS5SZXBvcnREZXBsb3ltZW50U3RhdHVzUmVxdWVzdC5VcGRhdGVIABI/CgZkZWxldGUYAiABKAsyLS5jdHJsLnYxLlJlcG9ydERlcGxveW1lbnRTdGF0dXNSZXF1ZXN0LkRlbGV0ZUgAGu0CCgZVcGRhdGUSEAoIazhzX25hbWUYASABKAkSSQoJaW5zdGFuY2VzGAIgAygLMjYuY3RybC52MS5SZXBvcnREZXBsb3ltZW50U3RhdHVzUmVxdWVzdC5VcGRhdGUuSW5zdGFuY2UahQIKCEluc3RhbmNlEhAKCGs4c19uYW1lGAEgASgJEg8KB2FkZHJlc3MYAiABKAkSFgoOY3B1X21pbGxpY29yZXMYAyABKAMSEgoKbWVtb3J5X21pYhgEIAEoAxJNCgZzdGF0dXMYBSABKA4yPS5jdHJsLnYxLlJlcG9ydERlcGxveW1lbnRTdGF0dXNSZXF1ZXN0LlVwZGF0ZS5JbnN0YW5jZS5TdGF0dXMiWwoGU3RhdHVzEhYKElNUQVRVU19VTlNQRUNJRklFRBAAEhIKDlNUQVRVU19QRU5ESU5HEAESEgoOU1RBVFVTX1JVTk5JTkcQAhIRCg1TVEFUVVNfRkFJTEVEEAMaGgoGRGVsZXRlEhAKCGs4c19uYW1lGAEgASgJQggKBmNoYW5nZSIgCh5SZXBvcnREZXBsb3ltZW50U3RhdHVzUmVzcG9uc2UibAobUmVwb3J0U2VudGluZWxTdGF0dXNSZXF1ZXN0EhAKCGs4c19uYW1lGAEgASgJEhoKEmF2YWlsYWJsZV9yZXBsaWNhcxgCIAEoBRIfCgZoZWFsdGgYAyABKA4yDy5jdHJsLnYxLkhlYWx0aCIeChxSZXBvcnRTZW50aW5lbFN0YXR1c1Jlc3BvbnNlIkQKF1dhdGNoRGVwbG95bWVudHNSZXF1ZXN0Eg4KBnJlZ2lvbhgBIAEoCRIZChF2ZXJzaW9uX2xhc3Rfc2VlbhgCIAEoBCJCChVXYXRjaFNlbnRpbmVsc1JlcXVlc3QSDgoGcmVnaW9uGAEgASgJEhkKEXZlcnNpb25fbGFzdF9zZWVuGAIgASgEIn0KDVNlbnRpbmVsU3RhdGUSDwoHdmVyc2lvbhgDIAEoBBInCgVhcHBseRgBIAEoCzIWLmN0cmwudjEuQXBwbHlTZW50aW5lbEgAEikKBmRlbGV0ZRgCIAEoCzIXLmN0cmwudjEuRGVsZXRlU2VudGluZWxIAEIHCgVzdGF0ZSKDAQoPRGVwbG95bWVudFN0YXRlEg8KB3ZlcnNpb24YAyABKAQSKQoFYXBwbHkYASABKAsyGC5jdHJsLnYxLkFwcGx5RGVwbG95bWVudEgAEisKBmRlbGV0ZRgCIAEoCzIZLmN0cmwudjEuRGVsZXRlRGVwbG95bWVudEgAQgcKBXN0YXRlItUBCg1BcHBseVNlbnRpbmVsEhAKCGs4c19uYW1lGAEgASgJEhQKDHdvcmtzcGFjZV9pZBgCIAEoCRISCgpwcm9qZWN0X2lkGAMgASgJEg4KBmFwcF9pZBgEIAEoCRIWCg5lbnZpcm9ubWVudF9pZBgFIAEoCRITCgtzZW50aW5lbF9pZBgGIAEoCRINCgVpbWFnZRgHIAEoCRIQCghyZXBsaWNhcxgIIAEoBRIWCg5jcHVfbWlsbGljb3JlcxgJIAEoAxISCgptZW1vcnlfbWliGAogASgDIiIKDkRlbGV0ZVNlbnRpbmVsEhAKCGs4c19uYW1lGAEgASgJIp8DCg9BcHBseURlcGxveW1lbnQSFQoNazhzX25hbWVzcGFjZRgBIAEoCRIQCghrOHNfbmFtZRgCIAEoCRIUCgx3b3Jrc3BhY2VfaWQYAyABKAkSEgoKcHJvamVjdF9pZBgEIAEoCRIWCg5lbnZpcm9ubWVudF9pZBgFIAEoCRIVCg1kZXBsb3ltZW50X2lkGAYgASgJEg0KBWltYWdlGAcgASgJEhAKCHJlcGxpY2FzGAggASgFEhYKDmNwdV9taWxsaWNvcmVzGAkgASgDEhIKCm1lbW9yeV9taWIYCiABKAMSFQoIYnVpbGRfaWQYCyABKAlIAIgBARInCh9lbmNyeXB0ZWRfZW52aXJvbm1lbnRfdmFyaWFibGVzGAwgASgMEg8KB2NvbW1hbmQYDSADKAkSDAoEcG9ydBgOIAEoBRIXCg9zaHV0ZG93bl9zaWduYWwYDyABKAkSGAoLaGVhbHRoY2hlY2sYESABKAxIAYgBARIOCgZhcHBfaWQYEiABKAlCCwoJX2J1aWxkX2lkQg4KDF9oZWFsdGhjaGVjayI7ChBEZWxldGVEZXBsb3ltZW50EhUKDWs4c19uYW1lc3BhY2UYASABKAkSEAoIazhzX25hbWUYAiABKAkiNAoQSGVhcnRiZWF0UmVxdWVzdBIOCgZyZWdpb24YASABKAkSEAoIcGxhdGZvcm0YAiABKAkiEwoRSGVhcnRiZWF0UmVzcG9uc2UqXQoGSGVhbHRoEhYKEkhFQUxUSF9VTlNQRUNJRklFRBAAEhIKDkhFQUxUSF9IRUFMVEhZEAESFAoQSEVBTFRIX1VOSEVBTFRIWRACEhEKDUhFQUxUSF9QQVVTRUQQAzLsBgoOQ2x1c3RlclNlcnZpY2USUAoQV2F0Y2hEZXBsb3ltZW50cxIgLmN0cmwudjEuV2F0Y2hEZXBsb3ltZW50c1JlcXVlc3QaGC5jdHJsLnYxLkRlcGxveW1lbnRTdGF0ZTABEkoKDldhdGNoU2VudGluZWxzEh4uY3RybC52MS5XYXRjaFNlbnRpbmVsc1JlcXVlc3QaFi5jdHJsLnYxLlNlbnRpbmVsU3RhdGUwARJaChdHZXREZXNpcmVkU2VudGluZWxTdGF0ZRInLmN0cmwudjEuR2V0RGVzaXJlZFNlbnRpbmVsU3RhdGVSZXF1ZXN0GhYuY3RybC52MS5TZW50aW5lbFN0YXRlEmMKFFJlcG9ydFNlbnRpbmVsU3RhdHVzEiQuY3RybC52MS5SZXBvcnRTZW50aW5lbFN0YXR1c1JlcXVlc3QaJS5jdHJsLnYxLlJlcG9ydFNlbnRpbmVsU3RhdHVzUmVzcG9uc2USYAoZR2V0RGVzaXJlZERlcGxveW1lbnRTdGF0ZRIpLmN0cmwudjEuR2V0RGVzaXJlZERlcGxveW1lbnRTdGF0ZVJlcXVlc3QaGC5jdHJsLnYxLkRlcGxveW1lbnRTdGF0ZRJpChZSZXBvcnREZXBsb3ltZW50U3RhdHVzEiYuY3RybC52MS5SZXBvcnREZXBsb3ltZW50U3RhdHVzUmVxdWVzdBonLmN0cmwudjEuUmVwb3J0RGVwbG95bWVudFN0YXR1c1Jlc3BvbnNlEm0KGldhdGNoQ2lsaXVtTmV0d29ya1BvbGljaWVzEiouY3RybC52MS5XYXRjaENpbGl1bU5ldHdvcmtQb2xpY2llc1JlcXVlc3QaIS5jdHJsLnYxLkNpbGl1bU5ldHdvcmtQb2xpY3lTdGF0ZTABEnsKIkdldERlc2lyZWRDaWxpdW1OZXR3b3JrUG9saWN5U3RhdGUSMi5jdHJsLnYxLkdldERlc2lyZWRDaWxpdW1OZXR3b3JrUG9saWN5U3RhdGVSZXF1ZXN0GiEuY3RybC52MS5DaWxpdW1OZXR3b3JrUG9saWN5U3RhdGUSQgoJSGVhcnRiZWF0EhkuY3RybC52MS5IZWFydGJlYXRSZXF1ZXN0GhouY3RybC52MS5IZWFydGJlYXRSZXNwb25zZUKLAQoLY29tLmN0cmwudjFCDENsdXN0ZXJQcm90b1ABWjFnaXRodWIuY29tL3Vua2V5ZWQvdW5rZXkvZ2VuL3Byb3RvL2N0cmwvdjE7Y3RybHYxogIDQ1hYqgIHQ3RybC5WMcoCB0N0cmxcVjHiAhNDdHJsXFYxXEdQQk1ldGFkYXRh6gIIQ3RybDo6VjFiBnByb3RvMw"); /** * @generated from message ctrl.v1.WatchCiliumNetworkPoliciesRequest @@ -575,37 +575,44 @@ export type ApplySentinel = Message<"ctrl.v1.ApplySentinel"> & { */ projectId: string; + /** + * app_id identifies the app within the project. + * + * @generated from field: string app_id = 4; + */ + appId: string; + /** * environment_id in which the sentinel should exist. * - * @generated from field: string environment_id = 4; + * @generated from field: string environment_id = 5; */ environmentId: string; /** * sentinel_id is the unique identifier for this sentinel globally * - * @generated from field: string sentinel_id = 5; + * @generated from field: string sentinel_id = 6; */ sentinelId: string; /** - * @generated from field: string image = 6; + * @generated from field: string image = 7; */ image: string; /** - * @generated from field: int32 replicas = 7; + * @generated from field: int32 replicas = 8; */ replicas: number; /** - * @generated from field: int64 cpu_millicores = 8; + * @generated from field: int64 cpu_millicores = 9; */ cpuMillicores: bigint; /** - * @generated from field: int64 memory_mib = 9; + * @generated from field: int64 memory_mib = 10; */ memoryMib: bigint; }; From f05b4bf613ff858ba35d2e9740877f39493f921c Mon Sep 17 00:00:00 2001 From: chronark Date: Sat, 7 Mar 2026 20:36:27 +0100 Subject: [PATCH 19/24] fix: step conflict --- ...lk_deployment_step_insert.sql_generated.go | 3 ++- .../deployment_step_insert.sql_generated.go | 6 +++-- pkg/db/querier_generated.go | 3 ++- pkg/db/queries/deployment_step_insert.sql | 5 +++- .../services/deployment/create_deployment.go | 24 ------------------- svc/sentinel/config.go | 2 +- 6 files changed, 13 insertions(+), 30 deletions(-) diff --git a/pkg/db/bulk_deployment_step_insert.sql_generated.go b/pkg/db/bulk_deployment_step_insert.sql_generated.go index 420c94caa29..5caa01f7fd4 100644 --- a/pkg/db/bulk_deployment_step_insert.sql_generated.go +++ b/pkg/db/bulk_deployment_step_insert.sql_generated.go @@ -9,7 +9,8 @@ import ( ) // bulkInsertDeploymentStep is the base query for bulk insert -const bulkInsertDeploymentStep = `INSERT INTO ` + "`" + `deployment_steps` + "`" + ` ( workspace_id, project_id, app_id, environment_id, deployment_id, step, started_at ) VALUES %s` +const bulkInsertDeploymentStep = `INSERT INTO ` + "`" + `deployment_steps` + "`" + ` ( workspace_id, project_id, app_id, environment_id, deployment_id, step, started_at ) VALUES %s ON DUPLICATE KEY UPDATE + started_at = VALUES(started_at)` // InsertDeploymentSteps performs bulk insert in a single query func (q *BulkQueries) InsertDeploymentSteps(ctx context.Context, db DBTX, args []InsertDeploymentStepParams) error { diff --git a/pkg/db/deployment_step_insert.sql_generated.go b/pkg/db/deployment_step_insert.sql_generated.go index bed3070df71..6d199fad48f 100644 --- a/pkg/db/deployment_step_insert.sql_generated.go +++ b/pkg/db/deployment_step_insert.sql_generated.go @@ -27,7 +27,8 @@ VALUES ( ?, ?, ? -) +) ON DUPLICATE KEY UPDATE + started_at = VALUES(started_at) ` type InsertDeploymentStepParams struct { @@ -59,7 +60,8 @@ type InsertDeploymentStepParams struct { // ?, // ?, // ? -// ) +// ) ON DUPLICATE KEY UPDATE +// started_at = VALUES(started_at) func (q *Queries) InsertDeploymentStep(ctx context.Context, db DBTX, arg InsertDeploymentStepParams) error { _, err := db.ExecContext(ctx, insertDeploymentStep, arg.WorkspaceID, diff --git a/pkg/db/querier_generated.go b/pkg/db/querier_generated.go index 0f37a8080c8..bb09d091aaf 100644 --- a/pkg/db/querier_generated.go +++ b/pkg/db/querier_generated.go @@ -1419,7 +1419,8 @@ type Querier interface { // ?, // ?, // ? - // ) + // ) ON DUPLICATE KEY UPDATE + // started_at = VALUES(started_at) InsertDeploymentStep(ctx context.Context, db DBTX, arg InsertDeploymentStepParams) error //InsertDeploymentTopology // diff --git a/pkg/db/queries/deployment_step_insert.sql b/pkg/db/queries/deployment_step_insert.sql index 600f185e71d..ff0f566b18e 100644 --- a/pkg/db/queries/deployment_step_insert.sql +++ b/pkg/db/queries/deployment_step_insert.sql @@ -16,4 +16,7 @@ VALUES ( sqlc.arg(deployment_id), sqlc.arg(step), sqlc.arg(started_at) -); +) ON DUPLICATE KEY UPDATE + started_at = VALUES(started_at) + +; diff --git a/svc/ctrl/services/deployment/create_deployment.go b/svc/ctrl/services/deployment/create_deployment.go index c8f4487879f..5fadd6eb1bc 100644 --- a/svc/ctrl/services/deployment/create_deployment.go +++ b/svc/ctrl/services/deployment/create_deployment.go @@ -148,10 +148,6 @@ func (s *Service) CreateDeployment( // Explicit docker image (CLI, REST API) gitBranch = branchFromGitCommit(req.Msg.GetGitCommit(), appWithSettings.App.DefaultBranch) - if tsErr := validateGitCommitTimestamp(req.Msg.GetGitCommit()); tsErr != nil { - return nil, tsErr - } - if gitCommit := req.Msg.GetGitCommit(); gitCommit != nil { gitCommitSha = gitCommit.GetCommitSha() gitCommitMessage = trimLength(gitCommit.GetCommitMessage(), maxCommitMessageLength) @@ -319,26 +315,6 @@ func branchFromGitCommit(gitCommit *ctrlv1.GitCommitInfo, defaultBranch string) return "main" } -// validateGitCommitTimestamp validates the timestamp in GitCommitInfo if present. -func validateGitCommitTimestamp(gitCommit *ctrlv1.GitCommitInfo) error { - if gitCommit == nil || gitCommit.GetTimestamp() == 0 { - return nil - } - timestamp := gitCommit.GetTimestamp() - - if timestamp < 1_000_000_000_000 { - return connect.NewError(connect.CodeInvalidArgument, - fmt.Errorf("git_commit_timestamp must be Unix epoch milliseconds, got %d (appears to be seconds format)", timestamp)) - } - - maxValidTimestamp := time.Now().Add(1 * time.Hour).UnixMilli() - if timestamp > maxValidTimestamp { - return connect.NewError(connect.CodeInvalidArgument, - fmt.Errorf("git_commit_timestamp %d is too far in the future (must be Unix epoch milliseconds)", timestamp)) - } - return nil -} - // buildDockerSource looks up the app's current deployment's Docker image and carries // over its git metadata for the new deployment record. func buildDockerSource( diff --git a/svc/sentinel/config.go b/svc/sentinel/config.go index 7ee81b189ac..adce2e45c66 100644 --- a/svc/sentinel/config.go +++ b/svc/sentinel/config.go @@ -44,7 +44,7 @@ type Config struct { EnvironmentID string `toml:"environment_id" config:"required,nonempty"` // Platform identifies the underlying cloud platform this sentinel is running on. - Platform string `toml:"platform" config:"required,nonempty,oneof=aws,local"` + Platform string `toml:"platform" config:"required,nonempty"` // Region is the geographic region identifier (e.g. "us-east-1"). // Included in structured logs and used for routing decisions. From 5c941a4e4f14ab09cf311b9010f958aa64d593e2 Mon Sep 17 00:00:00 2001 From: chronark Date: Sun, 8 Mar 2026 08:01:58 +0100 Subject: [PATCH 20/24] fix: rabbit --- pkg/db/bulk_deployment_step_insert.sql_generated.go | 4 +++- pkg/db/deployment_step_insert.sql_generated.go | 8 ++++++-- pkg/db/querier_generated.go | 4 +++- pkg/db/queries/deployment_step_insert.sql | 5 +++-- pkg/db/schema.sql | 1 - web/internal/db/src/schema/regions.ts | 2 +- 6 files changed, 16 insertions(+), 8 deletions(-) diff --git a/pkg/db/bulk_deployment_step_insert.sql_generated.go b/pkg/db/bulk_deployment_step_insert.sql_generated.go index 5caa01f7fd4..84b10a862a1 100644 --- a/pkg/db/bulk_deployment_step_insert.sql_generated.go +++ b/pkg/db/bulk_deployment_step_insert.sql_generated.go @@ -10,7 +10,9 @@ import ( // bulkInsertDeploymentStep is the base query for bulk insert const bulkInsertDeploymentStep = `INSERT INTO ` + "`" + `deployment_steps` + "`" + ` ( workspace_id, project_id, app_id, environment_id, deployment_id, step, started_at ) VALUES %s ON DUPLICATE KEY UPDATE - started_at = VALUES(started_at)` + started_at = VALUES(started_at), + ended_at = NULL, + error = NULL` // InsertDeploymentSteps performs bulk insert in a single query func (q *BulkQueries) InsertDeploymentSteps(ctx context.Context, db DBTX, args []InsertDeploymentStepParams) error { diff --git a/pkg/db/deployment_step_insert.sql_generated.go b/pkg/db/deployment_step_insert.sql_generated.go index 6d199fad48f..33eaeae5a66 100644 --- a/pkg/db/deployment_step_insert.sql_generated.go +++ b/pkg/db/deployment_step_insert.sql_generated.go @@ -28,7 +28,9 @@ VALUES ( ?, ? ) ON DUPLICATE KEY UPDATE - started_at = VALUES(started_at) + started_at = VALUES(started_at), + ended_at = NULL, + error = NULL ` type InsertDeploymentStepParams struct { @@ -61,7 +63,9 @@ type InsertDeploymentStepParams struct { // ?, // ? // ) ON DUPLICATE KEY UPDATE -// started_at = VALUES(started_at) +// started_at = VALUES(started_at), +// ended_at = NULL, +// error = NULL func (q *Queries) InsertDeploymentStep(ctx context.Context, db DBTX, arg InsertDeploymentStepParams) error { _, err := db.ExecContext(ctx, insertDeploymentStep, arg.WorkspaceID, diff --git a/pkg/db/querier_generated.go b/pkg/db/querier_generated.go index bb09d091aaf..227bf7e1b49 100644 --- a/pkg/db/querier_generated.go +++ b/pkg/db/querier_generated.go @@ -1420,7 +1420,9 @@ type Querier interface { // ?, // ? // ) ON DUPLICATE KEY UPDATE - // started_at = VALUES(started_at) + // started_at = VALUES(started_at), + // ended_at = NULL, + // error = NULL InsertDeploymentStep(ctx context.Context, db DBTX, arg InsertDeploymentStepParams) error //InsertDeploymentTopology // diff --git a/pkg/db/queries/deployment_step_insert.sql b/pkg/db/queries/deployment_step_insert.sql index ff0f566b18e..aac6af11806 100644 --- a/pkg/db/queries/deployment_step_insert.sql +++ b/pkg/db/queries/deployment_step_insert.sql @@ -17,6 +17,7 @@ VALUES ( sqlc.arg(step), sqlc.arg(started_at) ) ON DUPLICATE KEY UPDATE - started_at = VALUES(started_at) - + started_at = VALUES(started_at), + ended_at = NULL, + error = NULL ; diff --git a/pkg/db/schema.sql b/pkg/db/schema.sql index a497603b19f..4d9faaea818 100644 --- a/pkg/db/schema.sql +++ b/pkg/db/schema.sql @@ -736,7 +736,6 @@ CREATE TABLE `regions` ( `platform` varchar(64) NOT NULL, CONSTRAINT `regions_pk` PRIMARY KEY(`pk`), CONSTRAINT `regions_id_unique` UNIQUE(`id`), - CONSTRAINT `regions_name_unique` UNIQUE(`name`), CONSTRAINT `unique_region_per_platform` UNIQUE(`name`,`platform`) ); diff --git a/web/internal/db/src/schema/regions.ts b/web/internal/db/src/schema/regions.ts index 53c9398e702..d7bf1e76807 100644 --- a/web/internal/db/src/schema/regions.ts +++ b/web/internal/db/src/schema/regions.ts @@ -8,7 +8,7 @@ export const regions = mysqlTable( pk: bigint("pk", { mode: "number", unsigned: true }).autoincrement().primaryKey(), id: varchar("id", { length: 64 }).notNull().unique(), // e.g. us-east-1, us-west-2, etc. - name: varchar("name", { length: 64 }).unique().notNull(), + name: varchar("name", { length: 64 }).notNull(), // e.g. aws, gcp, azure, local, etc. platform: varchar("platform", { length: 64 }).notNull(), }, From 3099de61cb895eea0cc3c80a864cffe9779b907c Mon Sep 17 00:00:00 2001 From: chronark Date: Sun, 8 Mar 2026 08:22:43 +0100 Subject: [PATCH 21/24] fix: frontline uses correct platform/region --- dev/k8s/manifests/frontline.yaml | 3 ++- svc/frontline/config.go | 4 ++++ svc/frontline/routes/proxy/handler.go | 2 +- svc/frontline/run.go | 2 ++ svc/frontline/services/proxy/director.go | 1 + svc/frontline/services/proxy/forward.go | 1 + svc/frontline/services/proxy/headers.go | 1 + svc/frontline/services/proxy/interface.go | 3 ++- svc/frontline/services/proxy/service.go | 6 ++++-- svc/frontline/services/router/interface.go | 6 ++++-- svc/frontline/services/router/service.go | 20 +++++++++++++------- 11 files changed, 35 insertions(+), 14 deletions(-) diff --git a/dev/k8s/manifests/frontline.yaml b/dev/k8s/manifests/frontline.yaml index 3797aacd22d..479d1d1d784 100644 --- a/dev/k8s/manifests/frontline.yaml +++ b/dev/k8s/manifests/frontline.yaml @@ -6,7 +6,8 @@ metadata: namespace: unkey data: unkey.toml: | - region = "local.dev" + platform = "dev" + region = "local" challenge_port = 7070 http_port = 7443 apex_domain = "unkey.local" diff --git a/svc/frontline/config.go b/svc/frontline/config.go index 05b5365474f..85f67f7a063 100644 --- a/svc/frontline/config.go +++ b/svc/frontline/config.go @@ -30,6 +30,10 @@ type Config struct { // Serves general traffic over HTTPS by default. HttpPort int `toml:"http_port" config:"default=7443,min=1,max=65535"` + // Platform identifies the cloud provider + // ie: aws, gcp, local + Platform string `toml:"platform" config:"required"` + // Region identifies the geographic region where this node is deployed. // Used for observability, latency optimization, and cross-region routing. Region string `toml:"region" config:"required"` diff --git a/svc/frontline/routes/proxy/handler.go b/svc/frontline/routes/proxy/handler.go index fc8d67be3e2..8450d380ec2 100644 --- a/svc/frontline/routes/proxy/handler.go +++ b/svc/frontline/routes/proxy/handler.go @@ -48,5 +48,5 @@ func (h *Handler) Handle(ctx context.Context, sess *zen.Session) error { return h.ProxyService.ForwardToSentinel(ctx, sess, decision.LocalSentinel, decision.DeploymentID) } - return h.ProxyService.ForwardToRegion(ctx, sess, decision.NearestNLBRegion) + return h.ProxyService.ForwardToRegion(ctx, sess, decision.NearestNLBRegionPlatform) } diff --git a/svc/frontline/run.go b/svc/frontline/run.go index c661f70aec7..295704234e2 100644 --- a/svc/frontline/run.go +++ b/svc/frontline/run.go @@ -212,6 +212,7 @@ func Run(ctx context.Context, cfg Config) error { // Initialize router service routerSvc, err := router.New(router.Config{ + Platform: cfg.Platform, Region: cfg.Region, DB: db, FrontlineRouteCache: cache.FrontlineRoutes, @@ -225,6 +226,7 @@ func Run(ctx context.Context, cfg Config) error { // nolint:exhaustruct proxySvc, err := proxy.New(proxy.Config{ InstanceID: cfg.InstanceID, + Platform: cfg.Platform, Region: cfg.Region, ApexDomain: cfg.ApexDomain, Clock: clk, diff --git a/svc/frontline/services/proxy/director.go b/svc/frontline/services/proxy/director.go index 85c1ccfc8f0..73b416ec136 100644 --- a/svc/frontline/services/proxy/director.go +++ b/svc/frontline/services/proxy/director.go @@ -38,6 +38,7 @@ func (s *service) makeSentinelDirector(sess *zen.Session, deploymentID string, s func (s *service) makeRegionDirector(sess *zen.Session, startTime time.Time) func(*http.Request) { return func(req *http.Request) { req.Header.Set(HeaderFrontlineID, s.instanceID) + req.Header.Set(HeaderPlatform, s.platform) req.Header.Set(HeaderRegion, s.region) req.Header.Set(HeaderRequestID, sess.RequestID()) diff --git a/svc/frontline/services/proxy/forward.go b/svc/frontline/services/proxy/forward.go index 6bb0cf8b306..a161a98a208 100644 --- a/svc/frontline/services/proxy/forward.go +++ b/svc/frontline/services/proxy/forward.go @@ -29,6 +29,7 @@ type forwardConfig struct { func (s *service) forward(sess *zen.Session, cfg forwardConfig) error { sess.ResponseWriter().Header().Set(HeaderFrontlineID, s.instanceID) + sess.ResponseWriter().Header().Set(HeaderPlatform, s.platform) sess.ResponseWriter().Header().Set(HeaderRegion, s.region) sess.ResponseWriter().Header().Set(HeaderRequestID, sess.RequestID()) diff --git a/svc/frontline/services/proxy/headers.go b/svc/frontline/services/proxy/headers.go index 5add2b85100..246c22a572d 100644 --- a/svc/frontline/services/proxy/headers.go +++ b/svc/frontline/services/proxy/headers.go @@ -5,6 +5,7 @@ const ( // Headers set on BOTH response (to client) AND request (to downstream service) // These identify which frontline processed the request HeaderFrontlineID = "X-Unkey-Frontline-Id" // ID of the frontline instance + HeaderPlatform = "X-Unkey-Platform" // Platform of the frontline instance HeaderRegion = "X-Unkey-Region" // Region of the frontline instance HeaderRequestID = "X-Unkey-Request-Id" // Request ID for tracing diff --git a/svc/frontline/services/proxy/interface.go b/svc/frontline/services/proxy/interface.go index a2b32527e8e..0546a2652ac 100644 --- a/svc/frontline/services/proxy/interface.go +++ b/svc/frontline/services/proxy/interface.go @@ -21,7 +21,7 @@ type Service interface { // ForwardToRegion forwards a request to a remote region (HTTPS) // Keeps the original hostname so the remote frontline can do TLS termination and routing // Request start time is retrieved from context - ForwardToRegion(ctx context.Context, sess *zen.Session, targetRegion string) error + ForwardToRegion(ctx context.Context, sess *zen.Session, targetRegionPlatform string) error } // Config holds configuration for the proxy service. @@ -29,6 +29,7 @@ type Config struct { // InstanceID is the current frontline instance ID InstanceID string + Platform string // Region is the current frontline region Region string diff --git a/svc/frontline/services/proxy/service.go b/svc/frontline/services/proxy/service.go index b829d541744..c8db9bee5b1 100644 --- a/svc/frontline/services/proxy/service.go +++ b/svc/frontline/services/proxy/service.go @@ -22,6 +22,7 @@ import ( type service struct { instanceID string + platform string region string apexDomain string clock clock.Clock @@ -99,6 +100,7 @@ func New(cfg Config) (*service, error) { return &service{ instanceID: cfg.InstanceID, + platform: cfg.Platform, region: cfg.Region, apexDomain: cfg.ApexDomain, clock: cfg.Clock, @@ -129,7 +131,7 @@ func (s *service) ForwardToSentinel(ctx context.Context, sess *zen.Session, sent }) } -func (s *service) ForwardToRegion(ctx context.Context, sess *zen.Session, targetRegion string) error { +func (s *service) ForwardToRegion(ctx context.Context, sess *zen.Session, targetRegionPlatform string) error { startTime, _ := RequestStartTimeFromContext(ctx) if hopCountStr := sess.Request().Header.Get(HeaderFrontlineHops); hopCountStr != "" { @@ -148,7 +150,7 @@ func (s *service) ForwardToRegion(ctx context.Context, sess *zen.Session, target } } - targetURL, err := url.Parse(fmt.Sprintf("https://frontline.%s.%s", targetRegion, s.apexDomain)) + targetURL, err := url.Parse(fmt.Sprintf("https://frontline.%s.%s", targetRegionPlatform, s.apexDomain)) if err != nil { return fault.Wrap(err, fault.Code(codes.Frontline.Internal.InternalServerError.URN()), diff --git a/svc/frontline/services/router/interface.go b/svc/frontline/services/router/interface.go index f0ddf504ecd..c55d0712b77 100644 --- a/svc/frontline/services/router/interface.go +++ b/svc/frontline/services/router/interface.go @@ -11,8 +11,9 @@ type RouteDecision struct { // LocalSentinel is set if there's a healthy sentinel in the local region LocalSentinel *db.Sentinel - // NearestNLBRegion is set if we need to forward to another region's NLB - NearestNLBRegion string + // NearestNLBRegionPlatform is set if we need to forward to another region's NLB + // The format must be dns compatible and is typically "." + NearestNLBRegionPlatform string // DeploymentID to pass in X-Unkey-Deployment-Id header DeploymentID string @@ -24,6 +25,7 @@ type Service interface { } type Config struct { + Platform string Region string DB db.Database FrontlineRouteCache cache.Cache[string, db.FrontlineRoute] diff --git a/svc/frontline/services/router/service.go b/svc/frontline/services/router/service.go index e2c90aaba67..02eeaecf1fd 100644 --- a/svc/frontline/services/router/service.go +++ b/svc/frontline/services/router/service.go @@ -2,6 +2,7 @@ package router import ( "context" + "fmt" internalCaches "github.com/unkeyed/unkey/internal/services/caches" "github.com/unkeyed/unkey/pkg/cache" @@ -41,6 +42,7 @@ var regionProximity = map[string][]string{ } type service struct { + platform string region string db db.Database frontlineRouteCache cache.Cache[string, db.FrontlineRoute] @@ -51,6 +53,7 @@ var _ Service = (*service)(nil) func New(cfg Config) (*service, error) { return &service{ + platform: cfg.Platform, region: cfg.Region, db: cfg.DB, frontlineRouteCache: cfg.FrontlineRouteCache, @@ -95,9 +98,9 @@ func (s *service) LookupByHostname(ctx context.Context, hostname string) (*db.Fr func (s *service) SelectSentinel(route *db.FrontlineRoute, sentinels []db.Sentinel) (*RouteDecision, error) { decision := &RouteDecision{ - DeploymentID: route.DeploymentID, - LocalSentinel: nil, - NearestNLBRegion: "", + DeploymentID: route.DeploymentID, + LocalSentinel: nil, + NearestNLBRegionPlatform: "", } healthyByRegion := make(map[string]*db.Sentinel) @@ -123,16 +126,19 @@ func (s *service) SelectSentinel(route *db.FrontlineRoute, sentinels []db.Sentin return decision, nil } - nearestRegion := s.findNearestRegion(healthyByRegion) + nearestRegion := s.findNearestRegionPlatform(healthyByRegion) if nearestRegion != "" { - decision.NearestNLBRegion = nearestRegion + decision.NearestNLBRegionPlatform = nearestRegion } return decision, nil } -func (s *service) findNearestRegion(healthyByRegion map[string]*db.Sentinel) string { - proximityList, exists := regionProximity[s.region] +func (s *service) findNearestRegionPlatform(healthyByRegion map[string]*db.Sentinel) string { + + self := fmt.Sprintf("%s.%s", s.region, s.platform) + + proximityList, exists := regionProximity[self] if exists { for _, region := range proximityList { if _, ok := healthyByRegion[region]; ok { From 078a0451618d330c906f58297a8eb05ec7a26992 Mon Sep 17 00:00:00 2001 From: chronark Date: Sun, 8 Mar 2026 09:24:26 +0100 Subject: [PATCH 22/24] feat: add running state --- pkg/db/models_generated.go | 1 + pkg/db/schema.sql | 2 +- svc/ctrl/integration/harness.go | 4 ++-- svc/ctrl/services/cluster/rpc_watch_deployments.go | 2 +- svc/ctrl/worker/deploy/deploy_handler.go | 2 +- svc/ctrl/worker/deployment/deployment_state.go | 2 +- web/internal/db/src/schema/deployment_topology.ts | 1 + 7 files changed, 8 insertions(+), 6 deletions(-) diff --git a/pkg/db/models_generated.go b/pkg/db/models_generated.go index fbbe64ab9c3..3e343933ce2 100644 --- a/pkg/db/models_generated.go +++ b/pkg/db/models_generated.go @@ -366,6 +366,7 @@ const ( DeploymentTopologyDesiredStatusStarted DeploymentTopologyDesiredStatus = "started" DeploymentTopologyDesiredStatusStopping DeploymentTopologyDesiredStatus = "stopping" DeploymentTopologyDesiredStatusStopped DeploymentTopologyDesiredStatus = "stopped" + DeploymentTopologyDesiredStatusRunning DeploymentTopologyDesiredStatus = "running" ) func (e *DeploymentTopologyDesiredStatus) Scan(src interface{}) error { diff --git a/pkg/db/schema.sql b/pkg/db/schema.sql index 4d9faaea818..f0d8dbec4da 100644 --- a/pkg/db/schema.sql +++ b/pkg/db/schema.sql @@ -538,7 +538,7 @@ CREATE TABLE `deployment_topology` ( `region_id` varchar(64) NOT NULL DEFAULT '', `desired_replicas` int NOT NULL, `version` bigint unsigned NOT NULL, - `desired_status` enum('starting','started','stopping','stopped') NOT NULL, + `desired_status` enum('starting','started','stopping','stopped','running') NOT NULL, `created_at` bigint NOT NULL, `updated_at` bigint, CONSTRAINT `deployment_topology_pk` PRIMARY KEY(`pk`), diff --git a/svc/ctrl/integration/harness.go b/svc/ctrl/integration/harness.go index bd8eb887509..126a1958657 100644 --- a/svc/ctrl/integration/harness.go +++ b/svc/ctrl/integration/harness.go @@ -179,7 +179,7 @@ func (h *Harness) CreateDeployment(ctx context.Context, req CreateDeploymentRequ Region: req.Region, RegionID: regionID, DesiredReplicas: 1, - DesiredStatus: db.DeploymentTopologyDesiredStatusStarted, + DesiredStatus: db.DeploymentTopologyDesiredStatusRunning, Version: h.versionCounter, CreatedAt: h.Now(), }) @@ -197,7 +197,7 @@ func (h *Harness) CreateDeployment(ctx context.Context, req CreateDeploymentRequ Region: req.Region, RegionID: regionID, DesiredReplicas: 1, - DesiredStatus: db.DeploymentTopologyDesiredStatusStarted, + DesiredStatus: db.DeploymentTopologyDesiredStatusRunning, Version: h.versionCounter, CreatedAt: h.Now(), UpdatedAt: sql.NullInt64{Valid: false}, diff --git a/svc/ctrl/services/cluster/rpc_watch_deployments.go b/svc/ctrl/services/cluster/rpc_watch_deployments.go index 5c59007f948..350b5ff312c 100644 --- a/svc/ctrl/services/cluster/rpc_watch_deployments.go +++ b/svc/ctrl/services/cluster/rpc_watch_deployments.go @@ -117,7 +117,7 @@ func (s *Service) deploymentRowToState(row db.ListDeploymentTopologyByRegionRow) }, }, }, nil - case db.DeploymentTopologyDesiredStatusStarted, db.DeploymentTopologyDesiredStatusStarting: + case db.DeploymentTopologyDesiredStatusRunning, db.DeploymentTopologyDesiredStatusStarted, db.DeploymentTopologyDesiredStatusStarting: var buildID *string if row.Deployment.BuildID.Valid { buildID = &row.Deployment.BuildID.String diff --git a/svc/ctrl/worker/deploy/deploy_handler.go b/svc/ctrl/worker/deploy/deploy_handler.go index dda9ee21059..0360ce1e38f 100644 --- a/svc/ctrl/worker/deploy/deploy_handler.go +++ b/svc/ctrl/worker/deploy/deploy_handler.go @@ -427,7 +427,7 @@ func (w *Workflow) createTopologies( Region: rs.RegionName, RegionID: rs.RegionID, DesiredReplicas: replicas, - DesiredStatus: db.DeploymentTopologyDesiredStatusStarting, + DesiredStatus: db.DeploymentTopologyDesiredStatusRunning, Version: versionResp.GetVersion(), CreatedAt: time.Now().UnixMilli(), }) diff --git a/svc/ctrl/worker/deployment/deployment_state.go b/svc/ctrl/worker/deployment/deployment_state.go index bf060feeb8f..1a5c8278971 100644 --- a/svc/ctrl/worker/deployment/deployment_state.go +++ b/svc/ctrl/worker/deployment/deployment_state.go @@ -81,7 +81,7 @@ func (v *VirtualObject) ChangeDesiredState(ctx restate.ObjectContext, req *hydra switch req.GetState() { case hydrav1.DeploymentDesiredState_DEPLOYMENT_DESIRED_STATE_RUNNING: desiredState = db.DeploymentsDesiredStateRunning - topologyDesiredStatus = db.DeploymentTopologyDesiredStatusStarted + topologyDesiredStatus = db.DeploymentTopologyDesiredStatusRunning case hydrav1.DeploymentDesiredState_DEPLOYMENT_DESIRED_STATE_STANDBY: desiredState = db.DeploymentsDesiredStateStandby topologyDesiredStatus = db.DeploymentTopologyDesiredStatusStopped diff --git a/web/internal/db/src/schema/deployment_topology.ts b/web/internal/db/src/schema/deployment_topology.ts index 919dfe9293c..d91214fbe7c 100644 --- a/web/internal/db/src/schema/deployment_topology.ts +++ b/web/internal/db/src/schema/deployment_topology.ts @@ -36,6 +36,7 @@ export const deploymentTopology = mysqlTable( "started", "stopping", "stopped", + "running", ]).notNull(), ...lifecycleDates, }, From afbba9ab1ea1edaf08ea48cb45feb2e47dee0097 Mon Sep 17 00:00:00 2001 From: chronark Date: Sun, 8 Mar 2026 09:58:26 +0100 Subject: [PATCH 23/24] fix: relation key --- web/internal/db/src/schema/clusters.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web/internal/db/src/schema/clusters.ts b/web/internal/db/src/schema/clusters.ts index f3a150465df..d60812e4bfa 100644 --- a/web/internal/db/src/schema/clusters.ts +++ b/web/internal/db/src/schema/clusters.ts @@ -18,8 +18,8 @@ export const clusters = mysqlTable("clusters", { }).notNull(), }); -export const regionsRelations = relations(clusters, ({ one }) => ({ - workspace: one(regions, { +export const clustersRelations = relations(clusters, ({ one }) => ({ + region: one(regions, { fields: [clusters.regionId], references: [regions.id], }), From b52f848f983266b19fba6a7e836118b0066c955e Mon Sep 17 00:00:00 2001 From: chronark Date: Sun, 8 Mar 2026 09:58:50 +0100 Subject: [PATCH 24/24] fix: unique key --- web/internal/db/src/schema/deployment_topology.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/web/internal/db/src/schema/deployment_topology.ts b/web/internal/db/src/schema/deployment_topology.ts index d91214fbe7c..445dfe6debf 100644 --- a/web/internal/db/src/schema/deployment_topology.ts +++ b/web/internal/db/src/schema/deployment_topology.ts @@ -27,7 +27,7 @@ export const deploymentTopology = mysqlTable( // Version for state synchronization with edge agents. // Updated via Restate VersioningService on each mutation. // Edge agents track their last-seen version and request changes after it. - // Unique per region (composite index with region). + // Unique per regionId (composite index with regionId). version: bigint("version", { mode: "number", unsigned: true }).notNull(), // Deployment status @@ -41,8 +41,8 @@ export const deploymentTopology = mysqlTable( ...lifecycleDates, }, (table) => [ - uniqueIndex("unique_region_per_deployment").on(table.deploymentId, table.region), - uniqueIndex("unique_version_per_region").on(table.region, table.version), + uniqueIndex("unique_region_per_deployment").on(table.deploymentId, table.regionId), + uniqueIndex("unique_version_per_region").on(table.regionId, table.version), index("workspace_idx").on(table.workspaceId), index("status_idx").on(table.desiredStatus), ],