diff --git a/dev/config/ctrl-api.toml b/dev/config/ctrl-api.toml index fbb716d0f84..dfda5873950 100644 --- a/dev/config/ctrl-api.toml +++ b/dev/config/ctrl-api.toml @@ -2,7 +2,6 @@ instance_id = "ctrl-api-dev" region = "local" http_port = 7091 auth_token = "your-local-dev-key" -available_regions = ["local.dev"] default_domain = "unkey.local" cname_domain = "unkey.local" diff --git a/dev/config/krane.toml b/dev/config/krane.toml index 2e06a97eba7..68d4fef5e85 100644 --- a/dev/config/krane.toml +++ b/dev/config/krane.toml @@ -1,4 +1,5 @@ -region = "local.dev" +region = "local" +platform = "dev" [control_plane] url = "http://ctrl-api:7091" diff --git a/dev/k8s/manifests/ctrl-api.yaml b/dev/k8s/manifests/ctrl-api.yaml index 3f90aa3d386..22d3b502c71 100644 --- a/dev/k8s/manifests/ctrl-api.yaml +++ b/dev/k8s/manifests/ctrl-api.yaml @@ -12,7 +12,6 @@ data: region = "local" http_port = 7091 auth_token = "your-local-dev-key" - available_regions = ["local.dev"] default_domain = "unkey.local" cname_domain = "unkey.local" diff --git a/dev/k8s/manifests/ctrl-worker.yaml b/dev/k8s/manifests/ctrl-worker.yaml index d2fd62fcf72..bdefe0c050e 100644 --- a/dev/k8s/manifests/ctrl-worker.yaml +++ b/dev/k8s/manifests/ctrl-worker.yaml @@ -14,7 +14,6 @@ data: cname_domain = "unkey.local" build_platform = "linux/arm64" sentinel_image = "unkey/sentinel:latest" - available_regions = ["local.dev"] [database] primary = "unkey:password@tcp(mysql:3306)/unkey?parseTime=true&interpolateParams=true" diff --git a/dev/k8s/manifests/frontline.yaml b/dev/k8s/manifests/frontline.yaml index 3797aacd22d..479d1d1d784 100644 --- a/dev/k8s/manifests/frontline.yaml +++ b/dev/k8s/manifests/frontline.yaml @@ -6,7 +6,8 @@ metadata: namespace: unkey data: unkey.toml: | - region = "local.dev" + platform = "dev" + region = "local" challenge_port = 7070 http_port = 7443 apex_domain = "unkey.local" diff --git a/dev/k8s/manifests/krane.yaml b/dev/k8s/manifests/krane.yaml index 9436204cf13..a67cd40fb52 100644 --- a/dev/k8s/manifests/krane.yaml +++ b/dev/k8s/manifests/krane.yaml @@ -8,7 +8,8 @@ metadata: component: krane data: unkey.toml: | - region = "local.dev" + region = "local" + platform = "dev" [control] url = "http://ctrl-api:7091" diff --git a/docs/engineering/architecture/services/control-plane/api/configuration.mdx b/docs/engineering/architecture/services/control-plane/api/configuration.mdx index fc33416317e..398025cbf52 100644 --- a/docs/engineering/architecture/services/control-plane/api/configuration.mdx +++ b/docs/engineering/architecture/services/control-plane/api/configuration.mdx @@ -40,9 +40,7 @@ Rotation is manual today. There is no built-in rotation mechanism. TODO: Replace with JWT-based auth once `auth.unkey.cloud` is in place. - - Regions available for deployments. - + Base domain for wildcard certificates. @@ -117,7 +115,6 @@ prometheus_port = 9090 region = "${UNKEY_REGION}" instance_id = "${POD_NAME}" auth_token = "${UNKEY_AUTH_TOKEN}" -available_regions = ["eu-central-1.aws", "us-east-1.aws", "us-west-2.aws"] default_domain = "${UNKEY_DEFAULT_DOMAIN}" regional_domain = "${UNKEY_REGIONAL_DOMAIN}" cname_domain = "${UNKEY_CNAME_DOMAIN}" diff --git a/docs/engineering/architecture/services/control-plane/worker/configuration.mdx b/docs/engineering/architecture/services/control-plane/worker/configuration.mdx index c972d5572a0..76e47572999 100644 --- a/docs/engineering/architecture/services/control-plane/worker/configuration.mdx +++ b/docs/engineering/architecture/services/control-plane/worker/configuration.mdx @@ -38,7 +38,6 @@ These fields must be set for production deployments. | `default_domain` | string | `unkey.app` | Used for sentinel bootstrapping. | `build_platform` | string | `linux/amd64` | Build platform, format `linux/{arch}`. | `sentinel_image` | string | `ghcr.io/unkeyed/unkey:local` | Sentinel image override. -| `available_regions` | list | - | Regions allowed for deployments. | `acme` | object | - | ACME config for cert issuance. | `depot` | object | - | Depot.dev config for builds. | `registry` | object | - | Registry credentials for builds. @@ -126,7 +125,6 @@ instance_id = "${POD_NAME}" default_domain = "${UNKEY_DEFAULT_DOMAIN}" build_platform = "linux/amd64" sentinel_image = "ghcr.io/unkeyed/unkey:v2.0.77" -available_regions = ["eu-central-1.aws", "us-east-1.aws"] cname_domain = "${UNKEY_CNAME_DOMAIN}" [database] diff --git a/gen/proto/ctrl/v1/cluster.pb.go b/gen/proto/ctrl/v1/cluster.pb.go index 0091102f3c8..c1bd6254b3b 100644 --- a/gen/proto/ctrl/v1/cluster.pb.go +++ b/gen/proto/ctrl/v1/cluster.pb.go @@ -1084,14 +1084,16 @@ type ApplySentinel struct { WorkspaceId string `protobuf:"bytes,2,opt,name=workspace_id,json=workspaceId,proto3" json:"workspace_id,omitempty"` // project_id identifies the project within the workspace. ProjectId string `protobuf:"bytes,3,opt,name=project_id,json=projectId,proto3" json:"project_id,omitempty"` + // app_id identifies the app within the project. + AppId string `protobuf:"bytes,4,opt,name=app_id,json=appId,proto3" json:"app_id,omitempty"` // environment_id in which the sentinel should exist. - EnvironmentId string `protobuf:"bytes,4,opt,name=environment_id,json=environmentId,proto3" json:"environment_id,omitempty"` + EnvironmentId string `protobuf:"bytes,5,opt,name=environment_id,json=environmentId,proto3" json:"environment_id,omitempty"` // sentinel_id is the unique identifier for this sentinel globally - SentinelId string `protobuf:"bytes,5,opt,name=sentinel_id,json=sentinelId,proto3" json:"sentinel_id,omitempty"` - Image string `protobuf:"bytes,6,opt,name=image,proto3" json:"image,omitempty"` - Replicas int32 `protobuf:"varint,7,opt,name=replicas,proto3" json:"replicas,omitempty"` - CpuMillicores int64 `protobuf:"varint,8,opt,name=cpu_millicores,json=cpuMillicores,proto3" json:"cpu_millicores,omitempty"` - MemoryMib int64 `protobuf:"varint,9,opt,name=memory_mib,json=memoryMib,proto3" json:"memory_mib,omitempty"` + SentinelId string `protobuf:"bytes,6,opt,name=sentinel_id,json=sentinelId,proto3" json:"sentinel_id,omitempty"` + Image string `protobuf:"bytes,7,opt,name=image,proto3" json:"image,omitempty"` + Replicas int32 `protobuf:"varint,8,opt,name=replicas,proto3" json:"replicas,omitempty"` + CpuMillicores int64 `protobuf:"varint,9,opt,name=cpu_millicores,json=cpuMillicores,proto3" json:"cpu_millicores,omitempty"` + MemoryMib int64 `protobuf:"varint,10,opt,name=memory_mib,json=memoryMib,proto3" json:"memory_mib,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1147,6 +1149,13 @@ func (x *ApplySentinel) GetProjectId() string { return "" } +func (x *ApplySentinel) GetAppId() string { + if x != nil { + return x.AppId + } + return "" +} + func (x *ApplySentinel) GetEnvironmentId() string { if x != nil { return x.EnvironmentId @@ -1499,6 +1508,99 @@ func (x *DeleteDeployment) GetK8SName() string { return "" } +// HeartbeatRequest is sent periodically by krane agents to register their +// presence. The control plane uses this to populate regions and +// clusters tables. +type HeartbeatRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // region is the geographic region name (e.g. "us-east-1"). + Region string `protobuf:"bytes,1,opt,name=region,proto3" json:"region,omitempty"` + // platform is the infrastructure provider (e.g. "aws", "gcp", "local"). + Platform string `protobuf:"bytes,2,opt,name=platform,proto3" json:"platform,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *HeartbeatRequest) Reset() { + *x = HeartbeatRequest{} + mi := &file_ctrl_v1_cluster_proto_msgTypes[19] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *HeartbeatRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*HeartbeatRequest) ProtoMessage() {} + +func (x *HeartbeatRequest) ProtoReflect() protoreflect.Message { + mi := &file_ctrl_v1_cluster_proto_msgTypes[19] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use HeartbeatRequest.ProtoReflect.Descriptor instead. +func (*HeartbeatRequest) Descriptor() ([]byte, []int) { + return file_ctrl_v1_cluster_proto_rawDescGZIP(), []int{19} +} + +func (x *HeartbeatRequest) GetRegion() string { + if x != nil { + return x.Region + } + return "" +} + +func (x *HeartbeatRequest) GetPlatform() string { + if x != nil { + return x.Platform + } + return "" +} + +type HeartbeatResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *HeartbeatResponse) Reset() { + *x = HeartbeatResponse{} + mi := &file_ctrl_v1_cluster_proto_msgTypes[20] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *HeartbeatResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*HeartbeatResponse) ProtoMessage() {} + +func (x *HeartbeatResponse) ProtoReflect() protoreflect.Message { + mi := &file_ctrl_v1_cluster_proto_msgTypes[20] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use HeartbeatResponse.ProtoReflect.Descriptor instead. +func (*HeartbeatResponse) Descriptor() ([]byte, []int) { + return file_ctrl_v1_cluster_proto_rawDescGZIP(), []int{20} +} + type ReportDeploymentStatusRequest_Update struct { state protoimpl.MessageState `protogen:"open.v1"` K8SName string `protobuf:"bytes,1,opt,name=k8s_name,json=k8sName,proto3" json:"k8s_name,omitempty"` @@ -1509,7 +1611,7 @@ type ReportDeploymentStatusRequest_Update struct { func (x *ReportDeploymentStatusRequest_Update) Reset() { *x = ReportDeploymentStatusRequest_Update{} - mi := &file_ctrl_v1_cluster_proto_msgTypes[19] + mi := &file_ctrl_v1_cluster_proto_msgTypes[21] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1521,7 +1623,7 @@ func (x *ReportDeploymentStatusRequest_Update) String() string { func (*ReportDeploymentStatusRequest_Update) ProtoMessage() {} func (x *ReportDeploymentStatusRequest_Update) ProtoReflect() protoreflect.Message { - mi := &file_ctrl_v1_cluster_proto_msgTypes[19] + mi := &file_ctrl_v1_cluster_proto_msgTypes[21] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1560,7 +1662,7 @@ type ReportDeploymentStatusRequest_Delete struct { func (x *ReportDeploymentStatusRequest_Delete) Reset() { *x = ReportDeploymentStatusRequest_Delete{} - mi := &file_ctrl_v1_cluster_proto_msgTypes[20] + mi := &file_ctrl_v1_cluster_proto_msgTypes[22] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1572,7 +1674,7 @@ func (x *ReportDeploymentStatusRequest_Delete) String() string { func (*ReportDeploymentStatusRequest_Delete) ProtoMessage() {} func (x *ReportDeploymentStatusRequest_Delete) ProtoReflect() protoreflect.Message { - mi := &file_ctrl_v1_cluster_proto_msgTypes[20] + mi := &file_ctrl_v1_cluster_proto_msgTypes[22] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1608,7 +1710,7 @@ type ReportDeploymentStatusRequest_Update_Instance struct { func (x *ReportDeploymentStatusRequest_Update_Instance) Reset() { *x = ReportDeploymentStatusRequest_Update_Instance{} - mi := &file_ctrl_v1_cluster_proto_msgTypes[21] + mi := &file_ctrl_v1_cluster_proto_msgTypes[23] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1620,7 +1722,7 @@ func (x *ReportDeploymentStatusRequest_Update_Instance) String() string { func (*ReportDeploymentStatusRequest_Update_Instance) ProtoMessage() {} func (x *ReportDeploymentStatusRequest_Update_Instance) ProtoReflect() protoreflect.Message { - mi := &file_ctrl_v1_cluster_proto_msgTypes[21] + mi := &file_ctrl_v1_cluster_proto_msgTypes[23] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1741,20 +1843,22 @@ const file_ctrl_v1_cluster_proto_rawDesc = "" + "\aversion\x18\x03 \x01(\x04R\aversion\x120\n" + "\x05apply\x18\x01 \x01(\v2\x18.ctrl.v1.ApplyDeploymentH\x00R\x05apply\x123\n" + "\x06delete\x18\x02 \x01(\v2\x19.ctrl.v1.DeleteDeploymentH\x00R\x06deleteB\a\n" + - "\x05state\"\xac\x02\n" + + "\x05state\"\xc3\x02\n" + "\rApplySentinel\x12\x19\n" + "\bk8s_name\x18\x01 \x01(\tR\ak8sName\x12!\n" + "\fworkspace_id\x18\x02 \x01(\tR\vworkspaceId\x12\x1d\n" + "\n" + - "project_id\x18\x03 \x01(\tR\tprojectId\x12%\n" + - "\x0eenvironment_id\x18\x04 \x01(\tR\renvironmentId\x12\x1f\n" + - "\vsentinel_id\x18\x05 \x01(\tR\n" + + "project_id\x18\x03 \x01(\tR\tprojectId\x12\x15\n" + + "\x06app_id\x18\x04 \x01(\tR\x05appId\x12%\n" + + "\x0eenvironment_id\x18\x05 \x01(\tR\renvironmentId\x12\x1f\n" + + "\vsentinel_id\x18\x06 \x01(\tR\n" + "sentinelId\x12\x14\n" + - "\x05image\x18\x06 \x01(\tR\x05image\x12\x1a\n" + - "\breplicas\x18\a \x01(\x05R\breplicas\x12%\n" + - "\x0ecpu_millicores\x18\b \x01(\x03R\rcpuMillicores\x12\x1d\n" + + "\x05image\x18\a \x01(\tR\x05image\x12\x1a\n" + + "\breplicas\x18\b \x01(\x05R\breplicas\x12%\n" + + "\x0ecpu_millicores\x18\t \x01(\x03R\rcpuMillicores\x12\x1d\n" + "\n" + - "memory_mib\x18\t \x01(\x03R\tmemoryMib\"+\n" + + "memory_mib\x18\n" + + " \x01(\x03R\tmemoryMib\"+\n" + "\x0eDeleteSentinel\x12\x19\n" + "\bk8s_name\x18\x01 \x01(\tR\ak8sName\"\xf1\x04\n" + "\x0fApplyDeployment\x12#\n" + @@ -1782,12 +1886,16 @@ const file_ctrl_v1_cluster_proto_rawDesc = "" + "\f_healthcheck\"R\n" + "\x10DeleteDeployment\x12#\n" + "\rk8s_namespace\x18\x01 \x01(\tR\fk8sNamespace\x12\x19\n" + - "\bk8s_name\x18\x02 \x01(\tR\ak8sName*]\n" + + "\bk8s_name\x18\x02 \x01(\tR\ak8sName\"F\n" + + "\x10HeartbeatRequest\x12\x16\n" + + "\x06region\x18\x01 \x01(\tR\x06region\x12\x1a\n" + + "\bplatform\x18\x02 \x01(\tR\bplatform\"\x13\n" + + "\x11HeartbeatResponse*]\n" + "\x06Health\x12\x16\n" + "\x12HEALTH_UNSPECIFIED\x10\x00\x12\x12\n" + "\x0eHEALTH_HEALTHY\x10\x01\x12\x14\n" + "\x10HEALTH_UNHEALTHY\x10\x02\x12\x11\n" + - "\rHEALTH_PAUSED\x10\x032\xa8\x06\n" + + "\rHEALTH_PAUSED\x10\x032\xec\x06\n" + "\x0eClusterService\x12P\n" + "\x10WatchDeployments\x12 .ctrl.v1.WatchDeploymentsRequest\x1a\x18.ctrl.v1.DeploymentState0\x01\x12J\n" + "\x0eWatchSentinels\x12\x1e.ctrl.v1.WatchSentinelsRequest\x1a\x16.ctrl.v1.SentinelState0\x01\x12Z\n" + @@ -1796,7 +1904,8 @@ const file_ctrl_v1_cluster_proto_rawDesc = "" + "\x19GetDesiredDeploymentState\x12).ctrl.v1.GetDesiredDeploymentStateRequest\x1a\x18.ctrl.v1.DeploymentState\x12i\n" + "\x16ReportDeploymentStatus\x12&.ctrl.v1.ReportDeploymentStatusRequest\x1a'.ctrl.v1.ReportDeploymentStatusResponse\x12m\n" + "\x1aWatchCiliumNetworkPolicies\x12*.ctrl.v1.WatchCiliumNetworkPoliciesRequest\x1a!.ctrl.v1.CiliumNetworkPolicyState0\x01\x12{\n" + - "\"GetDesiredCiliumNetworkPolicyState\x122.ctrl.v1.GetDesiredCiliumNetworkPolicyStateRequest\x1a!.ctrl.v1.CiliumNetworkPolicyStateB\x8b\x01\n" + + "\"GetDesiredCiliumNetworkPolicyState\x122.ctrl.v1.GetDesiredCiliumNetworkPolicyStateRequest\x1a!.ctrl.v1.CiliumNetworkPolicyState\x12B\n" + + "\tHeartbeat\x12\x19.ctrl.v1.HeartbeatRequest\x1a\x1a.ctrl.v1.HeartbeatResponseB\x8b\x01\n" + "\vcom.ctrl.v1B\fClusterProtoP\x01Z1github.com/unkeyed/unkey/gen/proto/ctrl/v1;ctrlv1\xa2\x02\x03CXX\xaa\x02\aCtrl.V1\xca\x02\aCtrl\\V1\xe2\x02\x13Ctrl\\V1\\GPBMetadata\xea\x02\bCtrl::V1b\x06proto3" var ( @@ -1812,7 +1921,7 @@ func file_ctrl_v1_cluster_proto_rawDescGZIP() []byte { } var file_ctrl_v1_cluster_proto_enumTypes = make([]protoimpl.EnumInfo, 2) -var file_ctrl_v1_cluster_proto_msgTypes = make([]protoimpl.MessageInfo, 22) +var file_ctrl_v1_cluster_proto_msgTypes = make([]protoimpl.MessageInfo, 24) var file_ctrl_v1_cluster_proto_goTypes = []any{ (Health)(0), // 0: ctrl.v1.Health (ReportDeploymentStatusRequest_Update_Instance_Status)(0), // 1: ctrl.v1.ReportDeploymentStatusRequest.Update.Instance.Status @@ -1835,21 +1944,23 @@ var file_ctrl_v1_cluster_proto_goTypes = []any{ (*DeleteSentinel)(nil), // 18: ctrl.v1.DeleteSentinel (*ApplyDeployment)(nil), // 19: ctrl.v1.ApplyDeployment (*DeleteDeployment)(nil), // 20: ctrl.v1.DeleteDeployment - (*ReportDeploymentStatusRequest_Update)(nil), // 21: ctrl.v1.ReportDeploymentStatusRequest.Update - (*ReportDeploymentStatusRequest_Delete)(nil), // 22: ctrl.v1.ReportDeploymentStatusRequest.Delete - (*ReportDeploymentStatusRequest_Update_Instance)(nil), // 23: ctrl.v1.ReportDeploymentStatusRequest.Update.Instance + (*HeartbeatRequest)(nil), // 21: ctrl.v1.HeartbeatRequest + (*HeartbeatResponse)(nil), // 22: ctrl.v1.HeartbeatResponse + (*ReportDeploymentStatusRequest_Update)(nil), // 23: ctrl.v1.ReportDeploymentStatusRequest.Update + (*ReportDeploymentStatusRequest_Delete)(nil), // 24: ctrl.v1.ReportDeploymentStatusRequest.Delete + (*ReportDeploymentStatusRequest_Update_Instance)(nil), // 25: ctrl.v1.ReportDeploymentStatusRequest.Update.Instance } var file_ctrl_v1_cluster_proto_depIdxs = []int32{ 3, // 0: ctrl.v1.CiliumNetworkPolicyState.apply:type_name -> ctrl.v1.ApplyCiliumNetworkPolicy 4, // 1: ctrl.v1.CiliumNetworkPolicyState.delete:type_name -> ctrl.v1.DeleteCiliumNetworkPolicy - 21, // 2: ctrl.v1.ReportDeploymentStatusRequest.update:type_name -> ctrl.v1.ReportDeploymentStatusRequest.Update - 22, // 3: ctrl.v1.ReportDeploymentStatusRequest.delete:type_name -> ctrl.v1.ReportDeploymentStatusRequest.Delete + 23, // 2: ctrl.v1.ReportDeploymentStatusRequest.update:type_name -> ctrl.v1.ReportDeploymentStatusRequest.Update + 24, // 3: ctrl.v1.ReportDeploymentStatusRequest.delete:type_name -> ctrl.v1.ReportDeploymentStatusRequest.Delete 0, // 4: ctrl.v1.ReportSentinelStatusRequest.health:type_name -> ctrl.v1.Health 17, // 5: ctrl.v1.SentinelState.apply:type_name -> ctrl.v1.ApplySentinel 18, // 6: ctrl.v1.SentinelState.delete:type_name -> ctrl.v1.DeleteSentinel 19, // 7: ctrl.v1.DeploymentState.apply:type_name -> ctrl.v1.ApplyDeployment 20, // 8: ctrl.v1.DeploymentState.delete:type_name -> ctrl.v1.DeleteDeployment - 23, // 9: ctrl.v1.ReportDeploymentStatusRequest.Update.instances:type_name -> ctrl.v1.ReportDeploymentStatusRequest.Update.Instance + 25, // 9: ctrl.v1.ReportDeploymentStatusRequest.Update.instances:type_name -> ctrl.v1.ReportDeploymentStatusRequest.Update.Instance 1, // 10: ctrl.v1.ReportDeploymentStatusRequest.Update.Instance.status:type_name -> ctrl.v1.ReportDeploymentStatusRequest.Update.Instance.Status 13, // 11: ctrl.v1.ClusterService.WatchDeployments:input_type -> ctrl.v1.WatchDeploymentsRequest 14, // 12: ctrl.v1.ClusterService.WatchSentinels:input_type -> ctrl.v1.WatchSentinelsRequest @@ -1859,16 +1970,18 @@ var file_ctrl_v1_cluster_proto_depIdxs = []int32{ 9, // 16: ctrl.v1.ClusterService.ReportDeploymentStatus:input_type -> ctrl.v1.ReportDeploymentStatusRequest 2, // 17: ctrl.v1.ClusterService.WatchCiliumNetworkPolicies:input_type -> ctrl.v1.WatchCiliumNetworkPoliciesRequest 6, // 18: ctrl.v1.ClusterService.GetDesiredCiliumNetworkPolicyState:input_type -> ctrl.v1.GetDesiredCiliumNetworkPolicyStateRequest - 16, // 19: ctrl.v1.ClusterService.WatchDeployments:output_type -> ctrl.v1.DeploymentState - 15, // 20: ctrl.v1.ClusterService.WatchSentinels:output_type -> ctrl.v1.SentinelState - 15, // 21: ctrl.v1.ClusterService.GetDesiredSentinelState:output_type -> ctrl.v1.SentinelState - 12, // 22: ctrl.v1.ClusterService.ReportSentinelStatus:output_type -> ctrl.v1.ReportSentinelStatusResponse - 16, // 23: ctrl.v1.ClusterService.GetDesiredDeploymentState:output_type -> ctrl.v1.DeploymentState - 10, // 24: ctrl.v1.ClusterService.ReportDeploymentStatus:output_type -> ctrl.v1.ReportDeploymentStatusResponse - 5, // 25: ctrl.v1.ClusterService.WatchCiliumNetworkPolicies:output_type -> ctrl.v1.CiliumNetworkPolicyState - 5, // 26: ctrl.v1.ClusterService.GetDesiredCiliumNetworkPolicyState:output_type -> ctrl.v1.CiliumNetworkPolicyState - 19, // [19:27] is the sub-list for method output_type - 11, // [11:19] is the sub-list for method input_type + 21, // 19: ctrl.v1.ClusterService.Heartbeat:input_type -> ctrl.v1.HeartbeatRequest + 16, // 20: ctrl.v1.ClusterService.WatchDeployments:output_type -> ctrl.v1.DeploymentState + 15, // 21: ctrl.v1.ClusterService.WatchSentinels:output_type -> ctrl.v1.SentinelState + 15, // 22: ctrl.v1.ClusterService.GetDesiredSentinelState:output_type -> ctrl.v1.SentinelState + 12, // 23: ctrl.v1.ClusterService.ReportSentinelStatus:output_type -> ctrl.v1.ReportSentinelStatusResponse + 16, // 24: ctrl.v1.ClusterService.GetDesiredDeploymentState:output_type -> ctrl.v1.DeploymentState + 10, // 25: ctrl.v1.ClusterService.ReportDeploymentStatus:output_type -> ctrl.v1.ReportDeploymentStatusResponse + 5, // 26: ctrl.v1.ClusterService.WatchCiliumNetworkPolicies:output_type -> ctrl.v1.CiliumNetworkPolicyState + 5, // 27: ctrl.v1.ClusterService.GetDesiredCiliumNetworkPolicyState:output_type -> ctrl.v1.CiliumNetworkPolicyState + 22, // 28: ctrl.v1.ClusterService.Heartbeat:output_type -> ctrl.v1.HeartbeatResponse + 20, // [20:29] is the sub-list for method output_type + 11, // [11:20] is the sub-list for method input_type 11, // [11:11] is the sub-list for extension type_name 11, // [11:11] is the sub-list for extension extendee 0, // [0:11] is the sub-list for field type_name @@ -1902,7 +2015,7 @@ func file_ctrl_v1_cluster_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_ctrl_v1_cluster_proto_rawDesc), len(file_ctrl_v1_cluster_proto_rawDesc)), NumEnums: 2, - NumMessages: 22, + NumMessages: 24, NumExtensions: 0, NumServices: 1, }, diff --git a/gen/proto/ctrl/v1/ctrlv1connect/cluster.connect.go b/gen/proto/ctrl/v1/ctrlv1connect/cluster.connect.go index 1aaddb026c5..955e733fed1 100644 --- a/gen/proto/ctrl/v1/ctrlv1connect/cluster.connect.go +++ b/gen/proto/ctrl/v1/ctrlv1connect/cluster.connect.go @@ -66,6 +66,9 @@ const ( // ClusterServiceGetDesiredCiliumNetworkPolicyStateProcedure is the fully-qualified name of the // ClusterService's GetDesiredCiliumNetworkPolicyState RPC. ClusterServiceGetDesiredCiliumNetworkPolicyStateProcedure = "/ctrl.v1.ClusterService/GetDesiredCiliumNetworkPolicyState" + // ClusterServiceHeartbeatProcedure is the fully-qualified name of the ClusterService's Heartbeat + // RPC. + ClusterServiceHeartbeatProcedure = "/ctrl.v1.ClusterService/Heartbeat" ) // ClusterServiceClient is a client for the ctrl.v1.ClusterService service. @@ -96,6 +99,10 @@ type ClusterServiceClient interface { // GetDesiredCiliumNetworkPolicyState returns the current desired state for a single Cilium policy. // Used by the resync loop to verify consistency for existing resources. GetDesiredCiliumNetworkPolicyState(context.Context, *connect.Request[v1.GetDesiredCiliumNetworkPolicyStateRequest]) (*connect.Response[v1.CiliumNetworkPolicyState], error) + // Heartbeat is called periodically by krane agents to register their cluster + // and region with the control plane. This populates the regions and + // clusters tables, making regions dynamically discoverable. + Heartbeat(context.Context, *connect.Request[v1.HeartbeatRequest]) (*connect.Response[v1.HeartbeatResponse], error) } // NewClusterServiceClient constructs a client for the ctrl.v1.ClusterService service. By default, @@ -157,6 +164,12 @@ func NewClusterServiceClient(httpClient connect.HTTPClient, baseURL string, opts connect.WithSchema(clusterServiceMethods.ByName("GetDesiredCiliumNetworkPolicyState")), connect.WithClientOptions(opts...), ), + heartbeat: connect.NewClient[v1.HeartbeatRequest, v1.HeartbeatResponse]( + httpClient, + baseURL+ClusterServiceHeartbeatProcedure, + connect.WithSchema(clusterServiceMethods.ByName("Heartbeat")), + connect.WithClientOptions(opts...), + ), } } @@ -170,6 +183,7 @@ type clusterServiceClient struct { reportDeploymentStatus *connect.Client[v1.ReportDeploymentStatusRequest, v1.ReportDeploymentStatusResponse] watchCiliumNetworkPolicies *connect.Client[v1.WatchCiliumNetworkPoliciesRequest, v1.CiliumNetworkPolicyState] getDesiredCiliumNetworkPolicyState *connect.Client[v1.GetDesiredCiliumNetworkPolicyStateRequest, v1.CiliumNetworkPolicyState] + heartbeat *connect.Client[v1.HeartbeatRequest, v1.HeartbeatResponse] } // WatchDeployments calls ctrl.v1.ClusterService.WatchDeployments. @@ -213,6 +227,11 @@ func (c *clusterServiceClient) GetDesiredCiliumNetworkPolicyState(ctx context.Co return c.getDesiredCiliumNetworkPolicyState.CallUnary(ctx, req) } +// Heartbeat calls ctrl.v1.ClusterService.Heartbeat. +func (c *clusterServiceClient) Heartbeat(ctx context.Context, req *connect.Request[v1.HeartbeatRequest]) (*connect.Response[v1.HeartbeatResponse], error) { + return c.heartbeat.CallUnary(ctx, req) +} + // ClusterServiceHandler is an implementation of the ctrl.v1.ClusterService service. type ClusterServiceHandler interface { // WatchDeployments streams deployment state changes from the control plane to agents. @@ -241,6 +260,10 @@ type ClusterServiceHandler interface { // GetDesiredCiliumNetworkPolicyState returns the current desired state for a single Cilium policy. // Used by the resync loop to verify consistency for existing resources. GetDesiredCiliumNetworkPolicyState(context.Context, *connect.Request[v1.GetDesiredCiliumNetworkPolicyStateRequest]) (*connect.Response[v1.CiliumNetworkPolicyState], error) + // Heartbeat is called periodically by krane agents to register their cluster + // and region with the control plane. This populates the regions and + // clusters tables, making regions dynamically discoverable. + Heartbeat(context.Context, *connect.Request[v1.HeartbeatRequest]) (*connect.Response[v1.HeartbeatResponse], error) } // NewClusterServiceHandler builds an HTTP handler from the service implementation. It returns the @@ -298,6 +321,12 @@ func NewClusterServiceHandler(svc ClusterServiceHandler, opts ...connect.Handler connect.WithSchema(clusterServiceMethods.ByName("GetDesiredCiliumNetworkPolicyState")), connect.WithHandlerOptions(opts...), ) + clusterServiceHeartbeatHandler := connect.NewUnaryHandler( + ClusterServiceHeartbeatProcedure, + svc.Heartbeat, + connect.WithSchema(clusterServiceMethods.ByName("Heartbeat")), + connect.WithHandlerOptions(opts...), + ) return "/ctrl.v1.ClusterService/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case ClusterServiceWatchDeploymentsProcedure: @@ -316,6 +345,8 @@ func NewClusterServiceHandler(svc ClusterServiceHandler, opts ...connect.Handler clusterServiceWatchCiliumNetworkPoliciesHandler.ServeHTTP(w, r) case ClusterServiceGetDesiredCiliumNetworkPolicyStateProcedure: clusterServiceGetDesiredCiliumNetworkPolicyStateHandler.ServeHTTP(w, r) + case ClusterServiceHeartbeatProcedure: + clusterServiceHeartbeatHandler.ServeHTTP(w, r) default: http.NotFound(w, r) } @@ -356,3 +387,7 @@ func (UnimplementedClusterServiceHandler) WatchCiliumNetworkPolicies(context.Con func (UnimplementedClusterServiceHandler) GetDesiredCiliumNetworkPolicyState(context.Context, *connect.Request[v1.GetDesiredCiliumNetworkPolicyStateRequest]) (*connect.Response[v1.CiliumNetworkPolicyState], error) { return nil, connect.NewError(connect.CodeUnimplemented, errors.New("ctrl.v1.ClusterService.GetDesiredCiliumNetworkPolicyState is not implemented")) } + +func (UnimplementedClusterServiceHandler) Heartbeat(context.Context, *connect.Request[v1.HeartbeatRequest]) (*connect.Response[v1.HeartbeatResponse], error) { + return nil, connect.NewError(connect.CodeUnimplemented, errors.New("ctrl.v1.ClusterService.Heartbeat is not implemented")) +} diff --git a/gen/rpc/ctrl/cluster_generated.go b/gen/rpc/ctrl/cluster_generated.go index 229faf9e616..267786f8ab0 100644 --- a/gen/rpc/ctrl/cluster_generated.go +++ b/gen/rpc/ctrl/cluster_generated.go @@ -22,6 +22,7 @@ type ClusterServiceClient interface { ReportDeploymentStatus(ctx context.Context, req *v1.ReportDeploymentStatusRequest) (*v1.ReportDeploymentStatusResponse, error) WatchCiliumNetworkPolicies(ctx context.Context, req *v1.WatchCiliumNetworkPoliciesRequest) (*connect.ServerStreamForClient[v1.CiliumNetworkPolicyState], error) GetDesiredCiliumNetworkPolicyState(ctx context.Context, req *v1.GetDesiredCiliumNetworkPolicyStateRequest) (*v1.CiliumNetworkPolicyState, error) + Heartbeat(ctx context.Context, req *v1.HeartbeatRequest) (*v1.HeartbeatResponse, error) } var _ ClusterServiceClient = (*ConnectClusterServiceClient)(nil) @@ -112,3 +113,16 @@ func (c *ConnectClusterServiceClient) GetDesiredCiliumNetworkPolicyState(ctx con } return resp.Msg, nil } + +func (c *ConnectClusterServiceClient) Heartbeat(ctx context.Context, req *v1.HeartbeatRequest) (*v1.HeartbeatResponse, error) { + ctx, span := tracing.Start(ctx, "ClusterService.Heartbeat") + defer span.End() + resp, err := c.inner.Heartbeat(ctx, connect.NewRequest(req)) + if err != nil { + if connect.CodeOf(err) != connect.CodeNotFound { + tracing.RecordError(span, err) + } + return nil, err + } + return resp.Msg, nil +} diff --git a/pkg/db/BUILD.bazel b/pkg/db/BUILD.bazel index 11560d8e127..b19494ad0ce 100644 --- a/pkg/db/BUILD.bazel +++ b/pkg/db/BUILD.bazel @@ -34,6 +34,7 @@ go_library( "app_find_with_settings.sql_generated.go", "app_insert.sql_generated.go", "app_list_by_project.sql_generated.go", + "app_regional_settings_find_by_app_and_env.sql_generated.go", "app_runtime_settings_find.sql_generated.go", "app_runtime_settings_upsert.sql_generated.go", "app_update_deployments.sql_generated.go", @@ -51,6 +52,8 @@ go_library( "bulk_certificate_insert.sql_generated.go", "bulk_cilium_network_policy_insert.sql_generated.go", "bulk_clickhouse_workspace_settings_insert.sql_generated.go", + "bulk_cluster_region_upsert.sql_generated.go", + "bulk_cluster_upsert.sql_generated.go", "bulk_custom_domain_insert.sql_generated.go", "bulk_custom_domain_upsert.sql_generated.go", "bulk_deployment_insert.sql_generated.go", @@ -96,6 +99,10 @@ go_library( "clickhouse_workspace_settings_find_by_workspace_id.sql_generated.go", "clickhouse_workspace_settings_insert.sql_generated.go", "clickhouse_workspace_settings_update_limits.sql_generated.go", + "cluster_region_find_by_name.sql_generated.go", + "cluster_region_list.sql_generated.go", + "cluster_region_upsert.sql_generated.go", + "cluster_upsert.sql_generated.go", "custom_domain_delete_by_id.sql_generated.go", "custom_domain_find_by_domain.sql_generated.go", "custom_domain_find_by_domain_or_wildcard.sql_generated.go", diff --git a/pkg/db/app_regional_settings_find_by_app_and_env.sql_generated.go b/pkg/db/app_regional_settings_find_by_app_and_env.sql_generated.go new file mode 100644 index 00000000000..6cbefa9999f --- /dev/null +++ b/pkg/db/app_regional_settings_find_by_app_and_env.sql_generated.go @@ -0,0 +1,65 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.30.0 +// source: app_regional_settings_find_by_app_and_env.sql + +package db + +import ( + "context" +) + +const findAppRegionalSettingsByAppAndEnv = `-- name: FindAppRegionalSettingsByAppAndEnv :many +SELECT + ars.region_id, + r.name AS region_name, + ars.replicas +FROM app_regional_settings ars +JOIN regions r ON r.id = ars.region_id +WHERE ars.app_id = ? + AND ars.environment_id = ? +` + +type FindAppRegionalSettingsByAppAndEnvParams struct { + AppID string `db:"app_id"` + EnvironmentID string `db:"environment_id"` +} + +type FindAppRegionalSettingsByAppAndEnvRow struct { + RegionID string `db:"region_id"` + RegionName string `db:"region_name"` + Replicas int32 `db:"replicas"` +} + +// FindAppRegionalSettingsByAppAndEnv +// +// SELECT +// ars.region_id, +// r.name AS region_name, +// ars.replicas +// FROM app_regional_settings ars +// JOIN regions r ON r.id = ars.region_id +// WHERE ars.app_id = ? +// AND ars.environment_id = ? +func (q *Queries) FindAppRegionalSettingsByAppAndEnv(ctx context.Context, db DBTX, arg FindAppRegionalSettingsByAppAndEnvParams) ([]FindAppRegionalSettingsByAppAndEnvRow, error) { + rows, err := db.QueryContext(ctx, findAppRegionalSettingsByAppAndEnv, arg.AppID, arg.EnvironmentID) + if err != nil { + return nil, err + } + defer rows.Close() + var items []FindAppRegionalSettingsByAppAndEnvRow + for rows.Next() { + var i FindAppRegionalSettingsByAppAndEnvRow + if err := rows.Scan(&i.RegionID, &i.RegionName, &i.Replicas); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} diff --git a/pkg/db/bulk_cluster_region_upsert.sql_generated.go b/pkg/db/bulk_cluster_region_upsert.sql_generated.go new file mode 100644 index 00000000000..57e53a31744 --- /dev/null +++ b/pkg/db/bulk_cluster_region_upsert.sql_generated.go @@ -0,0 +1,40 @@ +// Code generated by sqlc bulk insert plugin. DO NOT EDIT. + +package db + +import ( + "context" + "fmt" + "strings" +) + +// bulkUpsertRegion is the base query for bulk insert +const bulkUpsertRegion = `INSERT INTO regions ( id, name, platform ) VALUES %s ON DUPLICATE KEY UPDATE name = name` + +// UpsertRegion performs bulk insert in a single query +func (q *BulkQueries) UpsertRegion(ctx context.Context, db DBTX, args []UpsertRegionParams) error { + + if len(args) == 0 { + return nil + } + + // Build the bulk insert query + valueClauses := make([]string, len(args)) + for i := range args { + valueClauses[i] = "( ?, ?, ? )" + } + + bulkQuery := fmt.Sprintf(bulkUpsertRegion, strings.Join(valueClauses, ", ")) + + // Collect all arguments + var allArgs []any + for _, arg := range args { + allArgs = append(allArgs, arg.ID) + allArgs = append(allArgs, arg.Name) + allArgs = append(allArgs, arg.Platform) + } + + // Execute the bulk insert + _, err := db.ExecContext(ctx, bulkQuery, allArgs...) + return err +} diff --git a/pkg/db/bulk_cluster_upsert.sql_generated.go b/pkg/db/bulk_cluster_upsert.sql_generated.go new file mode 100644 index 00000000000..a58cf2ee13b --- /dev/null +++ b/pkg/db/bulk_cluster_upsert.sql_generated.go @@ -0,0 +1,46 @@ +// Code generated by sqlc bulk insert plugin. DO NOT EDIT. + +package db + +import ( + "context" + "fmt" + "strings" +) + +// bulkUpsertCluster is the base query for bulk insert +const bulkUpsertCluster = `INSERT INTO clusters ( id, region_id, last_heartbeat_at ) VALUES %s ON DUPLICATE KEY UPDATE + last_heartbeat_at = ?` + +// UpsertCluster performs bulk insert in a single query +func (q *BulkQueries) UpsertCluster(ctx context.Context, db DBTX, args []UpsertClusterParams) error { + + if len(args) == 0 { + return nil + } + + // Build the bulk insert query + valueClauses := make([]string, len(args)) + for i := range args { + valueClauses[i] = "( ?, ?, ? )" + } + + bulkQuery := fmt.Sprintf(bulkUpsertCluster, strings.Join(valueClauses, ", ")) + + // Collect all arguments + var allArgs []any + for _, arg := range args { + allArgs = append(allArgs, arg.ID) + allArgs = append(allArgs, arg.RegionID) + allArgs = append(allArgs, arg.LastHeartbeatAt) + } + + // Add ON DUPLICATE KEY UPDATE parameters (only once, not per row) + if len(args) > 0 { + allArgs = append(allArgs, args[0].LastHeartbeatAt) + } + + // Execute the bulk insert + _, err := db.ExecContext(ctx, bulkQuery, allArgs...) + return err +} diff --git a/pkg/db/bulk_deployment_step_insert.sql_generated.go b/pkg/db/bulk_deployment_step_insert.sql_generated.go index 420c94caa29..84b10a862a1 100644 --- a/pkg/db/bulk_deployment_step_insert.sql_generated.go +++ b/pkg/db/bulk_deployment_step_insert.sql_generated.go @@ -9,7 +9,10 @@ import ( ) // bulkInsertDeploymentStep is the base query for bulk insert -const bulkInsertDeploymentStep = `INSERT INTO ` + "`" + `deployment_steps` + "`" + ` ( workspace_id, project_id, app_id, environment_id, deployment_id, step, started_at ) VALUES %s` +const bulkInsertDeploymentStep = `INSERT INTO ` + "`" + `deployment_steps` + "`" + ` ( workspace_id, project_id, app_id, environment_id, deployment_id, step, started_at ) VALUES %s ON DUPLICATE KEY UPDATE + started_at = VALUES(started_at), + ended_at = NULL, + error = NULL` // InsertDeploymentSteps performs bulk insert in a single query func (q *BulkQueries) InsertDeploymentSteps(ctx context.Context, db DBTX, args []InsertDeploymentStepParams) error { diff --git a/pkg/db/bulk_deployment_topology_insert.sql_generated.go b/pkg/db/bulk_deployment_topology_insert.sql_generated.go index 47d2f8d66fc..49b4a1ccf03 100644 --- a/pkg/db/bulk_deployment_topology_insert.sql_generated.go +++ b/pkg/db/bulk_deployment_topology_insert.sql_generated.go @@ -9,7 +9,7 @@ import ( ) // bulkInsertDeploymentTopology is the base query for bulk insert -const bulkInsertDeploymentTopology = `INSERT INTO ` + "`" + `deployment_topology` + "`" + ` ( workspace_id, deployment_id, region, desired_replicas, desired_status, version, created_at ) VALUES %s` +const bulkInsertDeploymentTopology = `INSERT INTO ` + "`" + `deployment_topology` + "`" + ` ( workspace_id, deployment_id, region, region_id, desired_replicas, desired_status, version, created_at ) VALUES %s` // InsertDeploymentTopologies performs bulk insert in a single query func (q *BulkQueries) InsertDeploymentTopologies(ctx context.Context, db DBTX, args []InsertDeploymentTopologyParams) error { @@ -21,7 +21,7 @@ func (q *BulkQueries) InsertDeploymentTopologies(ctx context.Context, db DBTX, a // Build the bulk insert query valueClauses := make([]string, len(args)) for i := range args { - valueClauses[i] = "( ?, ?, ?, ?, ?, ?, ? )" + valueClauses[i] = "( ?, ?, ?, ?, ?, ?, ?, ? )" } bulkQuery := fmt.Sprintf(bulkInsertDeploymentTopology, strings.Join(valueClauses, ", ")) @@ -32,6 +32,7 @@ func (q *BulkQueries) InsertDeploymentTopologies(ctx context.Context, db DBTX, a allArgs = append(allArgs, arg.WorkspaceID) allArgs = append(allArgs, arg.DeploymentID) allArgs = append(allArgs, arg.Region) + allArgs = append(allArgs, arg.RegionID) allArgs = append(allArgs, arg.DesiredReplicas) allArgs = append(allArgs, arg.DesiredStatus) allArgs = append(allArgs, arg.Version) diff --git a/pkg/db/cluster_region_find_by_name.sql_generated.go b/pkg/db/cluster_region_find_by_name.sql_generated.go new file mode 100644 index 00000000000..216404ba66f --- /dev/null +++ b/pkg/db/cluster_region_find_by_name.sql_generated.go @@ -0,0 +1,29 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.30.0 +// source: cluster_region_find_by_name.sql + +package db + +import ( + "context" +) + +const findRegionByNameAndPlatform = `-- name: FindRegionByNameAndPlatform :one +SELECT id FROM regions WHERE name = ? AND platform = ? +` + +type FindRegionByNameAndPlatformParams struct { + Name string `db:"name"` + Platform string `db:"platform"` +} + +// FindRegionByNameAndPlatform +// +// SELECT id FROM regions WHERE name = ? AND platform = ? +func (q *Queries) FindRegionByNameAndPlatform(ctx context.Context, db DBTX, arg FindRegionByNameAndPlatformParams) (string, error) { + row := db.QueryRowContext(ctx, findRegionByNameAndPlatform, arg.Name, arg.Platform) + var id string + err := row.Scan(&id) + return id, err +} diff --git a/pkg/db/cluster_region_list.sql_generated.go b/pkg/db/cluster_region_list.sql_generated.go new file mode 100644 index 00000000000..dbbed0edea8 --- /dev/null +++ b/pkg/db/cluster_region_list.sql_generated.go @@ -0,0 +1,46 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.30.0 +// source: cluster_region_list.sql + +package db + +import ( + "context" +) + +const listRegions = `-- name: ListRegions :many +SELECT id, name, platform FROM regions +` + +type ListRegionsRow struct { + ID string `db:"id"` + Name string `db:"name"` + Platform string `db:"platform"` +} + +// ListRegions +// +// SELECT id, name, platform FROM regions +func (q *Queries) ListRegions(ctx context.Context, db DBTX) ([]ListRegionsRow, error) { + rows, err := db.QueryContext(ctx, listRegions) + if err != nil { + return nil, err + } + defer rows.Close() + var items []ListRegionsRow + for rows.Next() { + var i ListRegionsRow + if err := rows.Scan(&i.ID, &i.Name, &i.Platform); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} diff --git a/pkg/db/cluster_region_upsert.sql_generated.go b/pkg/db/cluster_region_upsert.sql_generated.go new file mode 100644 index 00000000000..d70020b0662 --- /dev/null +++ b/pkg/db/cluster_region_upsert.sql_generated.go @@ -0,0 +1,48 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.30.0 +// source: cluster_region_upsert.sql + +package db + +import ( + "context" +) + +const upsertRegion = `-- name: UpsertRegion :exec +INSERT INTO regions ( + id, + name, + platform +) +VALUES ( + ?, + ?, + ? +) +ON DUPLICATE KEY UPDATE name = name +` + +type UpsertRegionParams struct { + ID string `db:"id"` + Name string `db:"name"` + Platform string `db:"platform"` +} + +// Inserts a region or does nothing if it already exists. +// +// INSERT INTO regions ( +// id, +// name, +// platform +// ) +// VALUES ( +// ?, +// ?, +// ? +// ) +// ON DUPLICATE KEY UPDATE name = name +func (q *Queries) UpsertRegion(ctx context.Context, db DBTX, arg UpsertRegionParams) error { + _, err := db.ExecContext(ctx, upsertRegion, arg.ID, arg.Name, arg.Platform) + return err +} diff --git a/pkg/db/cluster_upsert.sql_generated.go b/pkg/db/cluster_upsert.sql_generated.go new file mode 100644 index 00000000000..fba77920795 --- /dev/null +++ b/pkg/db/cluster_upsert.sql_generated.go @@ -0,0 +1,55 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.30.0 +// source: cluster_upsert.sql + +package db + +import ( + "context" +) + +const upsertCluster = `-- name: UpsertCluster :exec +INSERT INTO clusters ( + id, + region_id, + last_heartbeat_at +) +VALUES ( + ?, + ?, + ? +) +ON DUPLICATE KEY UPDATE + last_heartbeat_at = ? +` + +type UpsertClusterParams struct { + ID string `db:"id"` + RegionID string `db:"region_id"` + LastHeartbeatAt uint64 `db:"last_heartbeat_at"` +} + +// Upserts a cluster by region_id. If the cluster already exists, updates the heartbeat timestamp. +// +// INSERT INTO clusters ( +// id, +// region_id, +// last_heartbeat_at +// ) +// VALUES ( +// ?, +// ?, +// ? +// ) +// ON DUPLICATE KEY UPDATE +// last_heartbeat_at = ? +func (q *Queries) UpsertCluster(ctx context.Context, db DBTX, arg UpsertClusterParams) error { + _, err := db.ExecContext(ctx, upsertCluster, + arg.ID, + arg.RegionID, + arg.LastHeartbeatAt, + arg.LastHeartbeatAt, + ) + return err +} diff --git a/pkg/db/deployment_step_insert.sql_generated.go b/pkg/db/deployment_step_insert.sql_generated.go index bed3070df71..33eaeae5a66 100644 --- a/pkg/db/deployment_step_insert.sql_generated.go +++ b/pkg/db/deployment_step_insert.sql_generated.go @@ -27,7 +27,10 @@ VALUES ( ?, ?, ? -) +) ON DUPLICATE KEY UPDATE + started_at = VALUES(started_at), + ended_at = NULL, + error = NULL ` type InsertDeploymentStepParams struct { @@ -59,7 +62,10 @@ type InsertDeploymentStepParams struct { // ?, // ?, // ? -// ) +// ) ON DUPLICATE KEY UPDATE +// started_at = VALUES(started_at), +// ended_at = NULL, +// error = NULL func (q *Queries) InsertDeploymentStep(ctx context.Context, db DBTX, arg InsertDeploymentStepParams) error { _, err := db.ExecContext(ctx, insertDeploymentStep, arg.WorkspaceID, diff --git a/pkg/db/deployment_topology_insert.sql_generated.go b/pkg/db/deployment_topology_insert.sql_generated.go index 1ab6dc8a1f1..15ea4b561e8 100644 --- a/pkg/db/deployment_topology_insert.sql_generated.go +++ b/pkg/db/deployment_topology_insert.sql_generated.go @@ -14,6 +14,7 @@ INSERT INTO ` + "`" + `deployment_topology` + "`" + ` ( workspace_id, deployment_id, region, + region_id, desired_replicas, desired_status, version, @@ -25,6 +26,7 @@ INSERT INTO ` + "`" + `deployment_topology` + "`" + ` ( ?, ?, ?, + ?, ? ) ` @@ -33,6 +35,7 @@ type InsertDeploymentTopologyParams struct { WorkspaceID string `db:"workspace_id"` DeploymentID string `db:"deployment_id"` Region string `db:"region"` + RegionID string `db:"region_id"` DesiredReplicas int32 `db:"desired_replicas"` DesiredStatus DeploymentTopologyDesiredStatus `db:"desired_status"` Version uint64 `db:"version"` @@ -45,6 +48,7 @@ type InsertDeploymentTopologyParams struct { // workspace_id, // deployment_id, // region, +// region_id, // desired_replicas, // desired_status, // version, @@ -56,6 +60,7 @@ type InsertDeploymentTopologyParams struct { // ?, // ?, // ?, +// ?, // ? // ) func (q *Queries) InsertDeploymentTopology(ctx context.Context, db DBTX, arg InsertDeploymentTopologyParams) error { @@ -63,6 +68,7 @@ func (q *Queries) InsertDeploymentTopology(ctx context.Context, db DBTX, arg Ins arg.WorkspaceID, arg.DeploymentID, arg.Region, + arg.RegionID, arg.DesiredReplicas, arg.DesiredStatus, arg.Version, diff --git a/pkg/db/deployment_topology_list_by_versions.sql_generated.go b/pkg/db/deployment_topology_list_by_versions.sql_generated.go index 680b3fb13e4..b9f1577a2ff 100644 --- a/pkg/db/deployment_topology_list_by_versions.sql_generated.go +++ b/pkg/db/deployment_topology_list_by_versions.sql_generated.go @@ -12,7 +12,7 @@ import ( const listDeploymentTopologyByRegion = `-- name: ListDeploymentTopologyByRegion :many SELECT - dt.pk, dt.workspace_id, dt.deployment_id, dt.region, dt.desired_replicas, dt.version, dt.desired_status, dt.created_at, dt.updated_at, + dt.pk, dt.workspace_id, dt.deployment_id, dt.region, dt.region_id, dt.desired_replicas, dt.version, dt.desired_status, dt.created_at, dt.updated_at, d.pk, d.id, d.k8s_name, d.workspace_id, d.project_id, d.environment_id, d.app_id, d.image, d.build_id, d.git_commit_sha, d.git_branch, d.git_commit_message, d.git_commit_author_handle, d.git_commit_author_avatar_url, d.git_commit_timestamp, d.sentinel_config, d.openapi_spec, d.cpu_millicores, d.memory_mib, d.desired_state, d.encrypted_environment_variables, d.command, d.port, d.shutdown_signal, d.healthcheck, d.status, d.created_at, d.updated_at, w.k8s_namespace FROM ` + "`" + `deployment_topology` + "`" + ` dt @@ -39,7 +39,7 @@ type ListDeploymentTopologyByRegionRow struct { // Used by WatchDeployments to stream deployment state changes to krane agents. // // SELECT -// dt.pk, dt.workspace_id, dt.deployment_id, dt.region, dt.desired_replicas, dt.version, dt.desired_status, dt.created_at, dt.updated_at, +// dt.pk, dt.workspace_id, dt.deployment_id, dt.region, dt.region_id, dt.desired_replicas, dt.version, dt.desired_status, dt.created_at, dt.updated_at, // d.pk, d.id, d.k8s_name, d.workspace_id, d.project_id, d.environment_id, d.app_id, d.image, d.build_id, d.git_commit_sha, d.git_branch, d.git_commit_message, d.git_commit_author_handle, d.git_commit_author_avatar_url, d.git_commit_timestamp, d.sentinel_config, d.openapi_spec, d.cpu_millicores, d.memory_mib, d.desired_state, d.encrypted_environment_variables, d.command, d.port, d.shutdown_signal, d.healthcheck, d.status, d.created_at, d.updated_at, // w.k8s_namespace // FROM `deployment_topology` dt @@ -62,6 +62,7 @@ func (q *Queries) ListDeploymentTopologyByRegion(ctx context.Context, db DBTX, a &i.DeploymentTopology.WorkspaceID, &i.DeploymentTopology.DeploymentID, &i.DeploymentTopology.Region, + &i.DeploymentTopology.RegionID, &i.DeploymentTopology.DesiredReplicas, &i.DeploymentTopology.Version, &i.DeploymentTopology.DesiredStatus, diff --git a/pkg/db/deployment_topology_list_desired.sql_generated.go b/pkg/db/deployment_topology_list_desired.sql_generated.go index 23524e3764b..5a699812302 100644 --- a/pkg/db/deployment_topology_list_desired.sql_generated.go +++ b/pkg/db/deployment_topology_list_desired.sql_generated.go @@ -12,7 +12,7 @@ import ( const listDesiredDeploymentTopology = `-- name: ListDesiredDeploymentTopology :many SELECT - dt.pk, dt.workspace_id, dt.deployment_id, dt.region, dt.desired_replicas, dt.version, dt.desired_status, dt.created_at, dt.updated_at, + dt.pk, dt.workspace_id, dt.deployment_id, dt.region, dt.region_id, dt.desired_replicas, dt.version, dt.desired_status, dt.created_at, dt.updated_at, d.pk, d.id, d.k8s_name, d.workspace_id, d.project_id, d.environment_id, d.app_id, d.image, d.build_id, d.git_commit_sha, d.git_branch, d.git_commit_message, d.git_commit_author_handle, d.git_commit_author_avatar_url, d.git_commit_timestamp, d.sentinel_config, d.openapi_spec, d.cpu_millicores, d.memory_mib, d.desired_state, d.encrypted_environment_variables, d.command, d.port, d.shutdown_signal, d.healthcheck, d.status, d.created_at, d.updated_at, w.k8s_namespace FROM ` + "`" + `deployment_topology` + "`" + ` dt @@ -42,7 +42,7 @@ type ListDesiredDeploymentTopologyRow struct { // Used during bootstrap to stream all running deployments to krane. // // SELECT -// dt.pk, dt.workspace_id, dt.deployment_id, dt.region, dt.desired_replicas, dt.version, dt.desired_status, dt.created_at, dt.updated_at, +// dt.pk, dt.workspace_id, dt.deployment_id, dt.region, dt.region_id, dt.desired_replicas, dt.version, dt.desired_status, dt.created_at, dt.updated_at, // d.pk, d.id, d.k8s_name, d.workspace_id, d.project_id, d.environment_id, d.app_id, d.image, d.build_id, d.git_commit_sha, d.git_branch, d.git_commit_message, d.git_commit_author_handle, d.git_commit_author_avatar_url, d.git_commit_timestamp, d.sentinel_config, d.openapi_spec, d.cpu_millicores, d.memory_mib, d.desired_state, d.encrypted_environment_variables, d.command, d.port, d.shutdown_signal, d.healthcheck, d.status, d.created_at, d.updated_at, // w.k8s_namespace // FROM `deployment_topology` dt @@ -73,6 +73,7 @@ func (q *Queries) ListDesiredDeploymentTopology(ctx context.Context, db DBTX, ar &i.DeploymentTopology.WorkspaceID, &i.DeploymentTopology.DeploymentID, &i.DeploymentTopology.Region, + &i.DeploymentTopology.RegionID, &i.DeploymentTopology.DesiredReplicas, &i.DeploymentTopology.Version, &i.DeploymentTopology.DesiredStatus, diff --git a/pkg/db/models_generated.go b/pkg/db/models_generated.go index 157aaf0a56f..3e343933ce2 100644 --- a/pkg/db/models_generated.go +++ b/pkg/db/models_generated.go @@ -366,6 +366,7 @@ const ( DeploymentTopologyDesiredStatusStarted DeploymentTopologyDesiredStatus = "started" DeploymentTopologyDesiredStatusStopping DeploymentTopologyDesiredStatus = "stopping" DeploymentTopologyDesiredStatusStopped DeploymentTopologyDesiredStatus = "stopped" + DeploymentTopologyDesiredStatusRunning DeploymentTopologyDesiredStatus = "running" ) func (e *DeploymentTopologyDesiredStatus) Scan(src interface{}) error { @@ -1002,6 +1003,18 @@ type AppEnvironmentVariable struct { UpdatedAt sql.NullInt64 `db:"updated_at"` } +type AppRegionalSetting struct { + Pk uint64 `db:"pk"` + WorkspaceID string `db:"workspace_id"` + AppID string `db:"app_id"` + EnvironmentID string `db:"environment_id"` + RegionID string `db:"region_id"` + Replicas int32 `db:"replicas"` + HorizontalAutoscalingPolicyID sql.NullString `db:"horizontal_autoscaling_policy_id"` + CreatedAt int64 `db:"created_at"` + UpdatedAt sql.NullInt64 `db:"updated_at"` +} + type AppRuntimeSetting struct { Pk uint64 `db:"pk"` WorkspaceID string `db:"workspace_id"` @@ -1107,6 +1120,13 @@ type ClickhouseWorkspaceSetting struct { UpdatedAt sql.NullInt64 `db:"updated_at"` } +type Cluster struct { + Pk uint64 `db:"pk"` + ID string `db:"id"` + RegionID string `db:"region_id"` + LastHeartbeatAt uint64 `db:"last_heartbeat_at"` +} + type CustomDomain struct { Pk uint64 `db:"pk"` ID string `db:"id"` @@ -1178,6 +1198,7 @@ type DeploymentTopology struct { WorkspaceID string `db:"workspace_id"` DeploymentID string `db:"deployment_id"` Region string `db:"region"` + RegionID string `db:"region_id"` DesiredReplicas int32 `db:"desired_replicas"` Version uint64 `db:"version"` DesiredStatus DeploymentTopologyDesiredStatus `db:"desired_status"` @@ -1240,6 +1261,19 @@ type GithubRepoConnection struct { UpdatedAt sql.NullInt64 `db:"updated_at"` } +type HorizontalAutoscalingPolicy struct { + Pk uint64 `db:"pk"` + ID string `db:"id"` + WorkspaceID string `db:"workspace_id"` + ReplicasMin int32 `db:"replicas_min"` + ReplicasMax int32 `db:"replicas_max"` + MemoryThreshold sql.NullInt16 `db:"memory_threshold"` + CpuThreshold sql.NullInt16 `db:"cpu_threshold"` + RpsThreshold sql.NullInt16 `db:"rps_threshold"` + CreatedAt int64 `db:"created_at"` + UpdatedAt sql.NullInt64 `db:"updated_at"` +} + type Identity struct { Pk uint64 `db:"pk"` ID string `db:"id"` @@ -1416,6 +1450,13 @@ type RatelimitOverride struct { DeletedAtM sql.NullInt64 `db:"deleted_at_m"` } +type Region struct { + Pk uint64 `db:"pk"` + ID string `db:"id"` + Name string `db:"name"` + Platform string `db:"platform"` +} + type Role struct { Pk uint64 `db:"pk"` ID string `db:"id"` diff --git a/pkg/db/querier_bulk_generated.go b/pkg/db/querier_bulk_generated.go index 92409f4b208..f2055074c90 100644 --- a/pkg/db/querier_bulk_generated.go +++ b/pkg/db/querier_bulk_generated.go @@ -17,6 +17,8 @@ type BulkQuerier interface { InsertCertificates(ctx context.Context, db DBTX, args []InsertCertificateParams) error InsertCiliumNetworkPolicies(ctx context.Context, db DBTX, args []InsertCiliumNetworkPolicyParams) error InsertClickhouseWorkspaceSettingses(ctx context.Context, db DBTX, args []InsertClickhouseWorkspaceSettingsParams) error + UpsertRegion(ctx context.Context, db DBTX, args []UpsertRegionParams) error + UpsertCluster(ctx context.Context, db DBTX, args []UpsertClusterParams) error InsertCustomDomains(ctx context.Context, db DBTX, args []InsertCustomDomainParams) error UpsertCustomDomain(ctx context.Context, db DBTX, args []UpsertCustomDomainParams) error InsertDeployments(ctx context.Context, db DBTX, args []InsertDeploymentParams) error diff --git a/pkg/db/querier_generated.go b/pkg/db/querier_generated.go index bd65dab2391..227bf7e1b49 100644 --- a/pkg/db/querier_generated.go +++ b/pkg/db/querier_generated.go @@ -209,6 +209,17 @@ type Querier interface { // WHERE app_id = ? // AND environment_id = ? FindAppEnvVarsByAppAndEnv(ctx context.Context, db DBTX, arg FindAppEnvVarsByAppAndEnvParams) ([]FindAppEnvVarsByAppAndEnvRow, error) + //FindAppRegionalSettingsByAppAndEnv + // + // SELECT + // ars.region_id, + // r.name AS region_name, + // ars.replicas + // FROM app_regional_settings ars + // JOIN regions r ON r.id = ars.region_id + // WHERE ars.app_id = ? + // AND ars.environment_id = ? + FindAppRegionalSettingsByAppAndEnv(ctx context.Context, db DBTX, arg FindAppRegionalSettingsByAppAndEnvParams) ([]FindAppRegionalSettingsByAppAndEnvRow, error) //FindAppRuntimeSettingsByAppAndEnv // // SELECT app_runtime_settings.pk, app_runtime_settings.workspace_id, app_runtime_settings.app_id, app_runtime_settings.environment_id, app_runtime_settings.port, app_runtime_settings.cpu_millicores, app_runtime_settings.memory_mib, app_runtime_settings.command, app_runtime_settings.healthcheck, app_runtime_settings.region_config, app_runtime_settings.shutdown_signal, app_runtime_settings.sentinel_config, app_runtime_settings.created_at, app_runtime_settings.updated_at @@ -1020,6 +1031,10 @@ type Querier interface { // AND namespace_id = ? // AND identifier = ? FindRatelimitOverrideByIdentifier(ctx context.Context, db DBTX, arg FindRatelimitOverrideByIdentifierParams) (RatelimitOverride, error) + //FindRegionByNameAndPlatform + // + // SELECT id FROM regions WHERE name = ? AND platform = ? + FindRegionByNameAndPlatform(ctx context.Context, db DBTX, arg FindRegionByNameAndPlatformParams) (string, error) // Finds a role record by its ID // Returns: The role record if found // @@ -1404,7 +1419,10 @@ type Querier interface { // ?, // ?, // ? - // ) + // ) ON DUPLICATE KEY UPDATE + // started_at = VALUES(started_at), + // ended_at = NULL, + // error = NULL InsertDeploymentStep(ctx context.Context, db DBTX, arg InsertDeploymentStepParams) error //InsertDeploymentTopology // @@ -1412,6 +1430,7 @@ type Querier interface { // workspace_id, // deployment_id, // region, + // region_id, // desired_replicas, // desired_status, // version, @@ -1423,6 +1442,7 @@ type Querier interface { // ?, // ?, // ?, + // ?, // ? // ) InsertDeploymentTopology(ctx context.Context, db DBTX, arg InsertDeploymentTopologyParams) error @@ -1891,7 +1911,7 @@ type Querier interface { // Used by WatchDeployments to stream deployment state changes to krane agents. // // SELECT - // dt.pk, dt.workspace_id, dt.deployment_id, dt.region, dt.desired_replicas, dt.version, dt.desired_status, dt.created_at, dt.updated_at, + // dt.pk, dt.workspace_id, dt.deployment_id, dt.region, dt.region_id, dt.desired_replicas, dt.version, dt.desired_status, dt.created_at, dt.updated_at, // d.pk, d.id, d.k8s_name, d.workspace_id, d.project_id, d.environment_id, d.app_id, d.image, d.build_id, d.git_commit_sha, d.git_branch, d.git_commit_message, d.git_commit_author_handle, d.git_commit_author_avatar_url, d.git_commit_timestamp, d.sentinel_config, d.openapi_spec, d.cpu_millicores, d.memory_mib, d.desired_state, d.encrypted_environment_variables, d.command, d.port, d.shutdown_signal, d.healthcheck, d.status, d.created_at, d.updated_at, // w.k8s_namespace // FROM `deployment_topology` dt @@ -1913,7 +1933,7 @@ type Querier interface { // Used during bootstrap to stream all running deployments to krane. // // SELECT - // dt.pk, dt.workspace_id, dt.deployment_id, dt.region, dt.desired_replicas, dt.version, dt.desired_status, dt.created_at, dt.updated_at, + // dt.pk, dt.workspace_id, dt.deployment_id, dt.region, dt.region_id, dt.desired_replicas, dt.version, dt.desired_status, dt.created_at, dt.updated_at, // d.pk, d.id, d.k8s_name, d.workspace_id, d.project_id, d.environment_id, d.app_id, d.image, d.build_id, d.git_commit_sha, d.git_branch, d.git_commit_message, d.git_commit_author_handle, d.git_commit_author_avatar_url, d.git_commit_timestamp, d.sentinel_config, d.openapi_spec, d.cpu_millicores, d.memory_mib, d.desired_state, d.encrypted_environment_variables, d.command, d.port, d.shutdown_signal, d.healthcheck, d.status, d.created_at, d.updated_at, // w.k8s_namespace // FROM `deployment_topology` dt @@ -2267,6 +2287,10 @@ type Querier interface { // WHERE key_id IN (/*SLICE:key_ids*/?) // ORDER BY key_id, id ListRatelimitsByKeyIDs(ctx context.Context, db DBTX, keyIds []sql.NullString) ([]ListRatelimitsByKeyIDsRow, error) + //ListRegions + // + // SELECT id, name, platform FROM regions + ListRegions(ctx context.Context, db DBTX) ([]ListRegionsRow, error) //ListRepoConnectionDeployContexts // // SELECT @@ -2824,6 +2848,21 @@ type Querier interface { // sentinel_config = VALUES(sentinel_config), // updated_at = VALUES(updated_at) UpsertAppRuntimeSettings(ctx context.Context, db DBTX, arg UpsertAppRuntimeSettingsParams) error + // Upserts a cluster by region_id. If the cluster already exists, updates the heartbeat timestamp. + // + // INSERT INTO clusters ( + // id, + // region_id, + // last_heartbeat_at + // ) + // VALUES ( + // ?, + // ?, + // ? + // ) + // ON DUPLICATE KEY UPDATE + // last_heartbeat_at = ? + UpsertCluster(ctx context.Context, db DBTX, arg UpsertClusterParams) error //UpsertCustomDomain // // INSERT INTO custom_domains ( @@ -2939,6 +2978,20 @@ type Querier interface { // ratelimit_api_limit = VALUES(ratelimit_api_limit), // ratelimit_api_duration = VALUES(ratelimit_api_duration) UpsertQuota(ctx context.Context, db DBTX, arg UpsertQuotaParams) error + // Inserts a region or does nothing if it already exists. + // + // INSERT INTO regions ( + // id, + // name, + // platform + // ) + // VALUES ( + // ?, + // ?, + // ? + // ) + // ON DUPLICATE KEY UPDATE name = name + UpsertRegion(ctx context.Context, db DBTX, arg UpsertRegionParams) error //UpsertWorkspace // // INSERT INTO workspaces ( diff --git a/pkg/db/queries/app_regional_settings_find_by_app_and_env.sql b/pkg/db/queries/app_regional_settings_find_by_app_and_env.sql new file mode 100644 index 00000000000..e31b29469c7 --- /dev/null +++ b/pkg/db/queries/app_regional_settings_find_by_app_and_env.sql @@ -0,0 +1,9 @@ +-- name: FindAppRegionalSettingsByAppAndEnv :many +SELECT + ars.region_id, + r.name AS region_name, + ars.replicas +FROM app_regional_settings ars +JOIN regions r ON r.id = ars.region_id +WHERE ars.app_id = sqlc.arg(app_id) + AND ars.environment_id = sqlc.arg(environment_id); diff --git a/pkg/db/queries/cluster_region_find_by_name.sql b/pkg/db/queries/cluster_region_find_by_name.sql new file mode 100644 index 00000000000..88423b9f607 --- /dev/null +++ b/pkg/db/queries/cluster_region_find_by_name.sql @@ -0,0 +1,2 @@ +-- name: FindRegionByNameAndPlatform :one +SELECT id FROM regions WHERE name = sqlc.arg(name) AND platform = sqlc.arg(platform); diff --git a/pkg/db/queries/cluster_region_list.sql b/pkg/db/queries/cluster_region_list.sql new file mode 100644 index 00000000000..58d7b226378 --- /dev/null +++ b/pkg/db/queries/cluster_region_list.sql @@ -0,0 +1,2 @@ +-- name: ListRegions :many +SELECT id, name, platform FROM regions; diff --git a/pkg/db/queries/cluster_region_upsert.sql b/pkg/db/queries/cluster_region_upsert.sql new file mode 100644 index 00000000000..b7c436fb259 --- /dev/null +++ b/pkg/db/queries/cluster_region_upsert.sql @@ -0,0 +1,13 @@ +-- name: UpsertRegion :exec +-- Inserts a region or does nothing if it already exists. +INSERT INTO regions ( + id, + name, + platform +) +VALUES ( + sqlc.arg(id), + sqlc.arg(name), + sqlc.arg(platform) +) +ON DUPLICATE KEY UPDATE name = name; diff --git a/pkg/db/queries/cluster_upsert.sql b/pkg/db/queries/cluster_upsert.sql new file mode 100644 index 00000000000..51a365f63a9 --- /dev/null +++ b/pkg/db/queries/cluster_upsert.sql @@ -0,0 +1,14 @@ +-- name: UpsertCluster :exec +-- Upserts a cluster by region_id. If the cluster already exists, updates the heartbeat timestamp. +INSERT INTO clusters ( + id, + region_id, + last_heartbeat_at +) +VALUES ( + sqlc.arg(id), + sqlc.arg(region_id), + sqlc.arg(last_heartbeat_at) +) +ON DUPLICATE KEY UPDATE + last_heartbeat_at = sqlc.arg(last_heartbeat_at); diff --git a/pkg/db/queries/deployment_step_insert.sql b/pkg/db/queries/deployment_step_insert.sql index 600f185e71d..aac6af11806 100644 --- a/pkg/db/queries/deployment_step_insert.sql +++ b/pkg/db/queries/deployment_step_insert.sql @@ -16,4 +16,8 @@ VALUES ( sqlc.arg(deployment_id), sqlc.arg(step), sqlc.arg(started_at) -); +) ON DUPLICATE KEY UPDATE + started_at = VALUES(started_at), + ended_at = NULL, + error = NULL +; diff --git a/pkg/db/queries/deployment_topology_insert.sql b/pkg/db/queries/deployment_topology_insert.sql index 37158bb00ba..0f416dd05de 100644 --- a/pkg/db/queries/deployment_topology_insert.sql +++ b/pkg/db/queries/deployment_topology_insert.sql @@ -3,6 +3,7 @@ INSERT INTO `deployment_topology` ( workspace_id, deployment_id, region, + region_id, desired_replicas, desired_status, version, @@ -11,6 +12,7 @@ INSERT INTO `deployment_topology` ( sqlc.arg(workspace_id), sqlc.arg(deployment_id), sqlc.arg(region), + sqlc.arg(region_id), sqlc.arg(desired_replicas), sqlc.arg(desired_status), sqlc.arg(version), diff --git a/pkg/db/schema.sql b/pkg/db/schema.sql index 63c25e1c5cc..f0d8dbec4da 100644 --- a/pkg/db/schema.sql +++ b/pkg/db/schema.sql @@ -448,6 +448,20 @@ CREATE TABLE `app_runtime_settings` ( CONSTRAINT `app_runtime_settings_app_env_idx` UNIQUE(`app_id`,`environment_id`) ); +CREATE TABLE `app_regional_settings` ( + `pk` bigint unsigned AUTO_INCREMENT NOT NULL, + `workspace_id` varchar(256) NOT NULL, + `app_id` varchar(64) NOT NULL, + `environment_id` varchar(128) NOT NULL, + `region_id` varchar(64) NOT NULL, + `replicas` int NOT NULL DEFAULT 1, + `horizontal_autoscaling_policy_id` varchar(64), + `created_at` bigint NOT NULL, + `updated_at` bigint, + CONSTRAINT `app_regional_settings_pk` PRIMARY KEY(`pk`), + CONSTRAINT `unique_app_env_region` UNIQUE(`app_id`,`environment_id`,`region_id`) +); + CREATE TABLE `app_environment_variables` ( `pk` bigint unsigned AUTO_INCREMENT NOT NULL, `id` varchar(128) NOT NULL, @@ -521,9 +535,10 @@ CREATE TABLE `deployment_topology` ( `workspace_id` varchar(64) NOT NULL, `deployment_id` varchar(64) NOT NULL, `region` varchar(64) NOT NULL, + `region_id` varchar(64) NOT NULL DEFAULT '', `desired_replicas` int NOT NULL, `version` bigint unsigned NOT NULL, - `desired_status` enum('starting','started','stopping','stopped') NOT NULL, + `desired_status` enum('starting','started','stopping','stopped','running') NOT NULL, `created_at` bigint NOT NULL, `updated_at` bigint, CONSTRAINT `deployment_topology_pk` PRIMARY KEY(`pk`), @@ -704,6 +719,41 @@ CREATE TABLE `cilium_network_policies` ( CONSTRAINT `unique_version_per_region` UNIQUE(`region`,`version`) ); +CREATE TABLE `clusters` ( + `pk` bigint unsigned AUTO_INCREMENT NOT NULL, + `id` varchar(64) NOT NULL, + `region_id` varchar(64) NOT NULL, + `last_heartbeat_at` bigint unsigned NOT NULL, + CONSTRAINT `clusters_pk` PRIMARY KEY(`pk`), + CONSTRAINT `clusters_id_unique` UNIQUE(`id`), + CONSTRAINT `clusters_region_id_unique` UNIQUE(`region_id`) +); + +CREATE TABLE `regions` ( + `pk` bigint unsigned AUTO_INCREMENT NOT NULL, + `id` varchar(64) NOT NULL, + `name` varchar(64) NOT NULL, + `platform` varchar(64) NOT NULL, + CONSTRAINT `regions_pk` PRIMARY KEY(`pk`), + CONSTRAINT `regions_id_unique` UNIQUE(`id`), + CONSTRAINT `unique_region_per_platform` UNIQUE(`name`,`platform`) +); + +CREATE TABLE `horizontal_autoscaling_policies` ( + `pk` bigint unsigned AUTO_INCREMENT NOT NULL, + `id` varchar(64) NOT NULL, + `workspace_id` varchar(256) NOT NULL, + `replicas_min` int NOT NULL, + `replicas_max` int NOT NULL, + `memory_threshold` tinyint, + `cpu_threshold` tinyint, + `rps_threshold` tinyint, + `created_at` bigint NOT NULL, + `updated_at` bigint, + CONSTRAINT `horizontal_autoscaling_policies_pk` PRIMARY KEY(`pk`), + CONSTRAINT `horizontal_autoscaling_policies_id_unique` UNIQUE(`id`) +); + CREATE INDEX `workspace_id_idx` ON `apis` (`workspace_id`); CREATE INDEX `workspace_id_idx` ON `roles` (`workspace_id`); CREATE INDEX `key_auth_id_deleted_at_idx` ON `keys` (`key_auth_id`,`deleted_at_m`); @@ -723,6 +773,7 @@ CREATE INDEX `bucket` ON `audit_log_target` (`bucket`); CREATE INDEX `id_idx` ON `audit_log_target` (`id`); CREATE INDEX `environments_project_idx` ON `environments` (`project_id`); CREATE INDEX `apps_workspace_idx` ON `apps` (`workspace_id`); +CREATE INDEX `workspace_idx` ON `app_regional_settings` (`workspace_id`); CREATE INDEX `workspace_idx` ON `deployments` (`workspace_id`); CREATE INDEX `project_idx` ON `deployments` (`project_id`); CREATE INDEX `status_idx` ON `deployments` (`status`); @@ -741,4 +792,5 @@ CREATE INDEX `idx_region` ON `instances` (`region`); CREATE INDEX `environment_id_idx` ON `frontline_routes` (`environment_id`); CREATE INDEX `deployment_id_idx` ON `frontline_routes` (`deployment_id`); CREATE INDEX `installation_id_idx` ON `github_repo_connections` (`installation_id`); +CREATE INDEX `workspace_idx` ON `horizontal_autoscaling_policies` (`workspace_id`); diff --git a/pkg/uid/prefix.go b/pkg/uid/prefix.go index 942b7ceb422..4edb4dcf685 100644 --- a/pkg/uid/prefix.go +++ b/pkg/uid/prefix.go @@ -18,8 +18,10 @@ const ( RatelimitPrefix Prefix = "rl" AuditLogPrefix Prefix = "log" InstancePrefix Prefix = "ins" - SentinelPrefix Prefix = "se" + SentinelPrefix Prefix = "s" CiliumNetworkPolicyPrefix Prefix = "net" + ClusterPrefix Prefix = "cls" + RegionPrefix Prefix = "rgn" OrgPrefix Prefix = "org" // Control plane prefixes diff --git a/svc/ctrl/api/certificate.go b/svc/ctrl/api/certificate.go index 3ef70383518..8b4b88b19a5 100644 --- a/svc/ctrl/api/certificate.go +++ b/svc/ctrl/api/certificate.go @@ -18,7 +18,6 @@ type certificateBootstrap struct { database db.Database defaultDomain string regionalDomain string - regions []string } // run bootstraps wildcard certificates for all configured domains and starts @@ -35,9 +34,14 @@ func (c *certificateBootstrap) run(ctx context.Context) { // Bootstrap per-region wildcards (e.g., *.us-west-2.aws.unkey.cloud) if c.regionalDomain != "" { - for _, region := range c.regions { - domain := fmt.Sprintf("*.%s.%s", region, c.regionalDomain) - c.bootstrapDomain(ctx, domain) + regions, err := db.Query.ListRegions(ctx, c.database.RO()) + if err != nil { + logger.Error("Failed to list regions for certificate bootstrap", "error", err) + } else { + for _, region := range regions { + domain := fmt.Sprintf("*.%s.%s", region.Name, c.regionalDomain) + c.bootstrapDomain(ctx, domain) + } } } } diff --git a/svc/ctrl/api/config.go b/svc/ctrl/api/config.go index 655f5814086..b576e9e032b 100644 --- a/svc/ctrl/api/config.go +++ b/svc/ctrl/api/config.go @@ -64,10 +64,6 @@ type Config struct { // Used by clients and services to authenticate with this control plane. AuthToken string `toml:"auth_token" config:"required,nonempty"` - // AvailableRegions is a list of available regions for deployments. - // Typically in the format "region.provider", ie "us-east-1.aws", "local.dev" - AvailableRegions []string `toml:"available_regions"` - // DefaultDomain is the fallback domain for system operations. // Used for wildcard certificate bootstrapping. When set, the API will // ensure a wildcard certificate exists for *.{DefaultDomain}. diff --git a/svc/ctrl/api/harness_test.go b/svc/ctrl/api/harness_test.go index e5980d43426..79d605202b3 100644 --- a/svc/ctrl/api/harness_test.go +++ b/svc/ctrl/api/harness_test.go @@ -93,14 +93,14 @@ func newWebhookHarness(t *testing.T, cfg webhookHarnessConfig) *webhookHarness { } apiConfig := Config{ - InstanceID: "test", - Region: "local", - HttpPort: ctrlPort, - PrometheusPort: 0, - AuthToken: "", - AvailableRegions: []string{"local.dev"}, - DefaultDomain: "", - RegionalDomain: "", + InstanceID: "test", + Region: "local", + HttpPort: ctrlPort, + PrometheusPort: 0, + AuthToken: "", + + DefaultDomain: "", + RegionalDomain: "", Database: config.DatabaseConfig{ Primary: mysqlCfg.DSN, ReadonlyReplica: "", diff --git a/svc/ctrl/api/run.go b/svc/ctrl/api/run.go index 35144a5eb74..83703ab0fe5 100644 --- a/svc/ctrl/api/run.go +++ b/svc/ctrl/api/run.go @@ -217,7 +217,6 @@ func Run(ctx context.Context, cfg Config) error { database: database, defaultDomain: cfg.DefaultDomain, regionalDomain: cfg.RegionalDomain, - regions: cfg.AvailableRegions, } r.Go(func(ctx context.Context) error { certBootstrap.run(ctx) diff --git a/svc/ctrl/integration/harness.go b/svc/ctrl/integration/harness.go index 3bce686de2c..126a1958657 100644 --- a/svc/ctrl/integration/harness.go +++ b/svc/ctrl/integration/harness.go @@ -157,13 +157,29 @@ func (h *Harness) CreateDeployment(ctx context.Context, req CreateDeploymentRequ _, err = h.DB.RW().ExecContext(ctx, "UPDATE deployments SET image = ? WHERE id = ?", "nginx:1.19", deploymentID) require.NoError(h.t, err) + // Ensure the region exists + regionID := uid.New(uid.RegionPrefix) + err = db.Query.UpsertRegion(ctx, h.DB.RW(), db.UpsertRegionParams{ + ID: regionID, + Name: req.Region, + Platform: "test", + }) + require.NoError(h.t, err) + + regionID, err = db.Query.FindRegionByNameAndPlatform(ctx, h.DB.RO(), db.FindRegionByNameAndPlatformParams{ + Name: req.Region, + Platform: "test", + }) + require.NoError(h.t, err) + h.versionCounter++ err = db.Query.InsertDeploymentTopology(ctx, h.DB.RW(), db.InsertDeploymentTopologyParams{ WorkspaceID: workspaceID, DeploymentID: deploymentID, Region: req.Region, + RegionID: regionID, DesiredReplicas: 1, - DesiredStatus: db.DeploymentTopologyDesiredStatusStarted, + DesiredStatus: db.DeploymentTopologyDesiredStatusRunning, Version: h.versionCounter, CreatedAt: h.Now(), }) @@ -179,8 +195,9 @@ func (h *Harness) CreateDeployment(ctx context.Context, req CreateDeploymentRequ WorkspaceID: workspaceID, DeploymentID: deploymentID, Region: req.Region, + RegionID: regionID, DesiredReplicas: 1, - DesiredStatus: db.DeploymentTopologyDesiredStatusStarted, + DesiredStatus: db.DeploymentTopologyDesiredStatusRunning, Version: h.versionCounter, CreatedAt: h.Now(), UpdatedAt: sql.NullInt64{Valid: false}, diff --git a/svc/ctrl/integration/harness/harness.go b/svc/ctrl/integration/harness/harness.go index d2a9348843d..d7fbc8e1bd6 100644 --- a/svc/ctrl/integration/harness/harness.go +++ b/svc/ctrl/integration/harness/harness.go @@ -170,12 +170,12 @@ func New(t *testing.T) *Harness { }) deploySvc := deploy.New(deploy.Config{ - DB: database, - Clickhouse: chClient, - DefaultDomain: "test.example.com", - Vault: vaultClient, - SentinelImage: "test-sentinel:latest", - AvailableRegions: []string{"us-east-1"}, + DB: database, + Clickhouse: chClient, + DefaultDomain: "test.example.com", + Vault: vaultClient, + SentinelImage: "test-sentinel:latest", + GitHub: nil, DepotConfig: deploy.DepotConfig{APIUrl: "", ProjectRegion: ""}, RegistryConfig: deploy.RegistryConfig{URL: "", Username: "", Password: ""}, diff --git a/svc/ctrl/proto/ctrl/v1/cluster.proto b/svc/ctrl/proto/ctrl/v1/cluster.proto index 89c67621c12..a4b780e1823 100644 --- a/svc/ctrl/proto/ctrl/v1/cluster.proto +++ b/svc/ctrl/proto/ctrl/v1/cluster.proto @@ -64,6 +64,11 @@ service ClusterService { // GetDesiredCiliumNetworkPolicyState returns the current desired state for a single Cilium policy. // Used by the resync loop to verify consistency for existing resources. rpc GetDesiredCiliumNetworkPolicyState(GetDesiredCiliumNetworkPolicyStateRequest) returns (CiliumNetworkPolicyState); + + // Heartbeat is called periodically by krane agents to register their cluster + // and region with the control plane. This populates the regions and + // clusters tables, making regions dynamically discoverable. + rpc Heartbeat(HeartbeatRequest) returns (HeartbeatResponse); } message WatchCiliumNetworkPoliciesRequest { @@ -227,16 +232,19 @@ message ApplySentinel { // project_id identifies the project within the workspace. string project_id = 3; + // app_id identifies the app within the project. + string app_id = 4; + // environment_id in which the sentinel should exist. - string environment_id = 4; + string environment_id = 5; // sentinel_id is the unique identifier for this sentinel globally - string sentinel_id = 5; + string sentinel_id = 6; - string image = 6; - int32 replicas = 7; - int64 cpu_millicores = 8; - int64 memory_mib = 9; + string image = 7; + int32 replicas = 8; + int64 cpu_millicores = 9; + int64 memory_mib = 10; } // DeleteSentinel identifies a sentinel to remove from the cluster. @@ -323,3 +331,16 @@ message DeleteDeployment { string k8s_namespace = 1; string k8s_name = 2; } + +// HeartbeatRequest is sent periodically by krane agents to register their +// presence. The control plane uses this to populate regions and +// clusters tables. +message HeartbeatRequest { + // region is the geographic region name (e.g. "us-east-1"). + string region = 1; + + // platform is the infrastructure provider (e.g. "aws", "gcp", "local"). + string platform = 2; +} + +message HeartbeatResponse {} diff --git a/svc/ctrl/services/cluster/BUILD.bazel b/svc/ctrl/services/cluster/BUILD.bazel index 572149f2de4..b16e6f0515f 100644 --- a/svc/ctrl/services/cluster/BUILD.bazel +++ b/svc/ctrl/services/cluster/BUILD.bazel @@ -8,6 +8,7 @@ go_library( "rpc_get_desired_cilium_network_policy_state.go", "rpc_get_desired_deployment_state.go", "rpc_get_desired_sentinel_state.go", + "rpc_heartbeat.go", "rpc_report_deployment_status.go", "rpc_report_sentinel_status.go", "rpc_watch_cilium_network_policies.go", diff --git a/svc/ctrl/services/cluster/rpc_get_desired_cilium_network_policy_state.go b/svc/ctrl/services/cluster/rpc_get_desired_cilium_network_policy_state.go index 19af6b56e9b..9b69486617c 100644 --- a/svc/ctrl/services/cluster/rpc_get_desired_cilium_network_policy_state.go +++ b/svc/ctrl/services/cluster/rpc_get_desired_cilium_network_policy_state.go @@ -13,8 +13,8 @@ import ( // in the caller's region. This is a point query alternative to [Service.WatchCiliumNetworkPolicies] // for cases where an agent needs to fetch state for a specific policy rather than streaming all changes. // -// Returns CodeUnauthenticated if bearer token is invalid, CodeInvalidArgument if the -// X-Krane-Region header is missing, CodeNotFound if no policy exists with the given ID +// Returns CodeUnauthenticated if bearer token is invalid, CodeInvalidArgument if +// region or platform is missing, CodeNotFound if no policy exists with the given ID // in the specified region, or CodeInternal for database errors. func (s *Service) GetDesiredCiliumNetworkPolicyState( ctx context.Context, @@ -25,7 +25,11 @@ func (s *Service) GetDesiredCiliumNetworkPolicyState( } region := req.Header().Get("X-Krane-Region") - if err := assert.NotEmpty(region, "region is required"); err != nil { + platform := req.Header().Get("X-Krane-Platform") + if err := assert.All( + assert.NotEmpty(region, "region is required"), + assert.NotEmpty(platform, "platform is required"), + ); err != nil { return nil, connect.NewError(connect.CodeInvalidArgument, err) } diff --git a/svc/ctrl/services/cluster/rpc_get_desired_deployment_state.go b/svc/ctrl/services/cluster/rpc_get_desired_deployment_state.go index 5dc78b594d3..0eec1352e5f 100644 --- a/svc/ctrl/services/cluster/rpc_get_desired_deployment_state.go +++ b/svc/ctrl/services/cluster/rpc_get_desired_deployment_state.go @@ -20,8 +20,8 @@ import ( // (for archived or standby states) based on the deployment_topology's desired_state in the // database. Unhandled desired states result in CodeInternal. // -// Returns CodeUnauthenticated if bearer token is invalid, CodeInvalidArgument if the -// X-Krane-Region header is missing, CodeNotFound if no deployment exists with the given +// Returns CodeUnauthenticated if bearer token is invalid, CodeInvalidArgument if +// region or platform is missing, CodeNotFound if no deployment exists with the given // ID in the specified region, or CodeInternal for database errors or unhandled states. func (s *Service) GetDesiredDeploymentState(ctx context.Context, req *connect.Request[ctrlv1.GetDesiredDeploymentStateRequest]) (*connect.Response[ctrlv1.DeploymentState], error) { @@ -30,7 +30,11 @@ func (s *Service) GetDesiredDeploymentState(ctx context.Context, req *connect.Re } region := req.Header().Get("X-Krane-Region") - if err := assert.NotEmpty(region, "region is required"); err != nil { + platform := req.Header().Get("X-Krane-Platform") + if err := assert.All( + assert.NotEmpty(region, "region is required"), + assert.NotEmpty(platform, "platform is required"), + ); err != nil { return nil, connect.NewError(connect.CodeInvalidArgument, err) } diff --git a/svc/ctrl/services/cluster/rpc_get_desired_sentinel_state.go b/svc/ctrl/services/cluster/rpc_get_desired_sentinel_state.go index 71647378ba4..4660b911962 100644 --- a/svc/ctrl/services/cluster/rpc_get_desired_sentinel_state.go +++ b/svc/ctrl/services/cluster/rpc_get_desired_sentinel_state.go @@ -19,8 +19,8 @@ import ( // (for archived or standby states) based on the sentinel's desired_state in the database. // Unhandled desired states result in CodeInternal. // -// Returns CodeUnauthenticated if bearer token is invalid, CodeInvalidArgument if the -// X-Krane-Region header is missing, CodeNotFound if no sentinel exists with the given ID, +// Returns CodeUnauthenticated if bearer token is invalid, CodeInvalidArgument if +// region or platform is missing, CodeNotFound if no sentinel exists with the given ID, // or CodeInternal for database errors or unhandled states. func (s *Service) GetDesiredSentinelState(ctx context.Context, req *connect.Request[ctrlv1.GetDesiredSentinelStateRequest]) (*connect.Response[ctrlv1.SentinelState], error) { @@ -29,7 +29,11 @@ func (s *Service) GetDesiredSentinelState(ctx context.Context, req *connect.Requ } region := req.Header().Get("X-Krane-Region") - if err := assert.NotEmpty(region, "region is required"); err != nil { + platform := req.Header().Get("X-Krane-Platform") + if err := assert.All( + assert.NotEmpty(region, "region is required"), + assert.NotEmpty(platform, "platform is required"), + ); err != nil { return nil, connect.NewError(connect.CodeInvalidArgument, err) } diff --git a/svc/ctrl/services/cluster/rpc_heartbeat.go b/svc/ctrl/services/cluster/rpc_heartbeat.go new file mode 100644 index 00000000000..0457df52400 --- /dev/null +++ b/svc/ctrl/services/cluster/rpc_heartbeat.go @@ -0,0 +1,68 @@ +package cluster + +import ( + "context" + "time" + + "connectrpc.com/connect" + ctrlv1 "github.com/unkeyed/unkey/gen/proto/ctrl/v1" + "github.com/unkeyed/unkey/pkg/assert" + "github.com/unkeyed/unkey/pkg/db" + "github.com/unkeyed/unkey/pkg/logger" + "github.com/unkeyed/unkey/pkg/uid" +) + +// Heartbeat registers or refreshes a krane agent's cluster and region in the +// control plane. Agents call this periodically so the control plane knows which +// regions are available. +// +// The method upserts into regions (keyed by region name) and clusters +// (keyed by region_id), updating the heartbeat timestamp on each call. +func (s *Service) Heartbeat(ctx context.Context, req *connect.Request[ctrlv1.HeartbeatRequest]) (*connect.Response[ctrlv1.HeartbeatResponse], error) { + if err := s.authenticate(req); err != nil { + return nil, err + } + + region := req.Msg.GetRegion() + platform := req.Msg.GetPlatform() + + if err := assert.All( + assert.NotEmpty(region, "region is required"), + assert.NotEmpty(platform, "platform is required"), + ); err != nil { + return nil, err + } + + now := time.Now().UnixMilli() + + err := db.Query.UpsertRegion(ctx, s.db.RW(), db.UpsertRegionParams{ + ID: uid.New(uid.RegionPrefix), + Name: region, + Platform: platform, + }) + if err != nil { + logger.Error("failed to upsert region", "error", err, "region", region) + return nil, err + } + + regionID, err := db.Query.FindRegionByNameAndPlatform(ctx, s.db.RW(), db.FindRegionByNameAndPlatformParams{ + Name: region, + Platform: platform, + }) + if err != nil { + logger.Error("failed to find region", "error", err, "region", region) + return nil, err + } + + err = db.Query.UpsertCluster(ctx, s.db.RW(), db.UpsertClusterParams{ + ID: uid.New(uid.ClusterPrefix), + RegionID: regionID, + LastHeartbeatAt: uint64(now), + }) + if err != nil { + logger.Error("failed to upsert cluster", "error", err, "region", region) + return nil, err + } + + return connect.NewResponse(&ctrlv1.HeartbeatResponse{}), nil +} diff --git a/svc/ctrl/services/cluster/rpc_report_deployment_status.go b/svc/ctrl/services/cluster/rpc_report_deployment_status.go index 07e07b1180d..19727811b90 100644 --- a/svc/ctrl/services/cluster/rpc_report_deployment_status.go +++ b/svc/ctrl/services/cluster/rpc_report_deployment_status.go @@ -33,15 +33,16 @@ func (s *Service) ReportDeploymentStatus(ctx context.Context, req *connect.Reque return nil, err } region := req.Header().Get("X-Krane-Region") + platform := req.Header().Get("X-Krane-Platform") - err := assert.All( + if err := assert.All( assert.NotEmpty(region, "region is required"), - ) - if err != nil { - return nil, err + assert.NotEmpty(platform, "platform is required"), + ); err != nil { + return nil, connect.NewError(connect.CodeInvalidArgument, err) } - err = db.TxRetry(ctx, s.db.RW(), func(txCtx context.Context, tx db.DBTX) error { + err := db.TxRetry(ctx, s.db.RW(), func(txCtx context.Context, tx db.DBTX) error { switch msg := req.Msg.GetChange().(type) { case *ctrlv1.ReportDeploymentStatusRequest_Update_: diff --git a/svc/ctrl/services/cluster/rpc_report_sentinel_status.go b/svc/ctrl/services/cluster/rpc_report_sentinel_status.go index b258c47740c..ef0ffeaa457 100644 --- a/svc/ctrl/services/cluster/rpc_report_sentinel_status.go +++ b/svc/ctrl/services/cluster/rpc_report_sentinel_status.go @@ -29,12 +29,13 @@ func (s *Service) ReportSentinelStatus(ctx context.Context, req *connect.Request } region := req.Header().Get("X-Krane-Region") + platform := req.Header().Get("X-Krane-Platform") - err := assert.All( + if err := assert.All( assert.NotEmpty(region, "region is required"), - ) - if err != nil { - return nil, err + assert.NotEmpty(platform, "platform is required"), + ); err != nil { + return nil, connect.NewError(connect.CodeInvalidArgument, err) } var health db.SentinelsHealth @@ -48,7 +49,7 @@ func (s *Service) ReportSentinelStatus(ctx context.Context, req *connect.Request case ctrlv1.Health_HEALTH_UNSPECIFIED: health = db.SentinelsHealthUnknown } - err = db.Query.UpdateSentinelAvailableReplicasAndHealth(ctx, s.db.RW(), db.UpdateSentinelAvailableReplicasAndHealthParams{ + err := db.Query.UpdateSentinelAvailableReplicasAndHealth(ctx, s.db.RW(), db.UpdateSentinelAvailableReplicasAndHealthParams{ K8sName: req.Msg.GetK8SName(), AvailableReplicas: req.Msg.GetAvailableReplicas(), Health: health, diff --git a/svc/ctrl/services/cluster/rpc_watch_cilium_network_policies.go b/svc/ctrl/services/cluster/rpc_watch_cilium_network_policies.go index 91f0c705d92..c7366a90aa9 100644 --- a/svc/ctrl/services/cluster/rpc_watch_cilium_network_policies.go +++ b/svc/ctrl/services/cluster/rpc_watch_cilium_network_policies.go @@ -34,8 +34,12 @@ func (s *Service) WatchCiliumNetworkPolicies( return err } - region := req.Msg.GetRegion() - if err := assert.NotEmpty(region, "region is required"); err != nil { + region := req.Header().Get("X-Krane-Region") + platform := req.Header().Get("X-Krane-Platform") + if err := assert.All( + assert.NotEmpty(region, "region is required"), + assert.NotEmpty(platform, "platform is required"), + ); err != nil { return connect.NewError(connect.CodeInvalidArgument, err) } diff --git a/svc/ctrl/services/cluster/rpc_watch_deployments.go b/svc/ctrl/services/cluster/rpc_watch_deployments.go index 585c9965774..350b5ff312c 100644 --- a/svc/ctrl/services/cluster/rpc_watch_deployments.go +++ b/svc/ctrl/services/cluster/rpc_watch_deployments.go @@ -37,8 +37,12 @@ func (s *Service) WatchDeployments( return err } - region := req.Msg.GetRegion() - if err := assert.NotEmpty(region, "region is required"); err != nil { + region := req.Header().Get("X-Krane-Region") + platform := req.Header().Get("X-Krane-Platform") + if err := assert.All( + assert.NotEmpty(region, "region is required"), + assert.NotEmpty(platform, "platform is required"), + ); err != nil { return connect.NewError(connect.CodeInvalidArgument, err) } @@ -113,7 +117,7 @@ func (s *Service) deploymentRowToState(row db.ListDeploymentTopologyByRegionRow) }, }, }, nil - case db.DeploymentTopologyDesiredStatusStarted, db.DeploymentTopologyDesiredStatusStarting: + case db.DeploymentTopologyDesiredStatusRunning, db.DeploymentTopologyDesiredStatusStarted, db.DeploymentTopologyDesiredStatusStarting: var buildID *string if row.Deployment.BuildID.Valid { buildID = &row.Deployment.BuildID.String diff --git a/svc/ctrl/services/cluster/rpc_watch_sentinels.go b/svc/ctrl/services/cluster/rpc_watch_sentinels.go index 53178254d52..0f79a678f86 100644 --- a/svc/ctrl/services/cluster/rpc_watch_sentinels.go +++ b/svc/ctrl/services/cluster/rpc_watch_sentinels.go @@ -34,8 +34,12 @@ func (s *Service) WatchSentinels( return err } - region := req.Msg.GetRegion() - if err := assert.NotEmpty(region, "region is required"); err != nil { + region := req.Header().Get("X-Krane-Region") + platform := req.Header().Get("X-Krane-Platform") + if err := assert.All( + assert.NotEmpty(region, "region is required"), + assert.NotEmpty(platform, "platform is required"), + ); err != nil { return connect.NewError(connect.CodeInvalidArgument, err) } versionCursor := req.Msg.GetVersionLastSeen() diff --git a/svc/ctrl/services/deployment/create_deployment.go b/svc/ctrl/services/deployment/create_deployment.go index c8f4487879f..5fadd6eb1bc 100644 --- a/svc/ctrl/services/deployment/create_deployment.go +++ b/svc/ctrl/services/deployment/create_deployment.go @@ -148,10 +148,6 @@ func (s *Service) CreateDeployment( // Explicit docker image (CLI, REST API) gitBranch = branchFromGitCommit(req.Msg.GetGitCommit(), appWithSettings.App.DefaultBranch) - if tsErr := validateGitCommitTimestamp(req.Msg.GetGitCommit()); tsErr != nil { - return nil, tsErr - } - if gitCommit := req.Msg.GetGitCommit(); gitCommit != nil { gitCommitSha = gitCommit.GetCommitSha() gitCommitMessage = trimLength(gitCommit.GetCommitMessage(), maxCommitMessageLength) @@ -319,26 +315,6 @@ func branchFromGitCommit(gitCommit *ctrlv1.GitCommitInfo, defaultBranch string) return "main" } -// validateGitCommitTimestamp validates the timestamp in GitCommitInfo if present. -func validateGitCommitTimestamp(gitCommit *ctrlv1.GitCommitInfo) error { - if gitCommit == nil || gitCommit.GetTimestamp() == 0 { - return nil - } - timestamp := gitCommit.GetTimestamp() - - if timestamp < 1_000_000_000_000 { - return connect.NewError(connect.CodeInvalidArgument, - fmt.Errorf("git_commit_timestamp must be Unix epoch milliseconds, got %d (appears to be seconds format)", timestamp)) - } - - maxValidTimestamp := time.Now().Add(1 * time.Hour).UnixMilli() - if timestamp > maxValidTimestamp { - return connect.NewError(connect.CodeInvalidArgument, - fmt.Errorf("git_commit_timestamp %d is too far in the future (must be Unix epoch milliseconds)", timestamp)) - } - return nil -} - // buildDockerSource looks up the app's current deployment's Docker image and carries // over its git metadata for the new deployment record. func buildDockerSource( diff --git a/svc/ctrl/services/deployment/doc.go b/svc/ctrl/services/deployment/doc.go index 6e36b2aa511..bd5decc9889 100644 --- a/svc/ctrl/services/deployment/doc.go +++ b/svc/ctrl/services/deployment/doc.go @@ -45,7 +45,7 @@ // svc := deployment.New(deployment.Config{ // Database: db, // Restate: restateClient, -// AvailableRegions: []string{"us-east-1", "eu-west-1"}, + // BuildStorage: s3Storage, // }) package deployment diff --git a/svc/ctrl/worker/config.go b/svc/ctrl/worker/config.go index e131f2204e1..5c8ef10b5e3 100644 --- a/svc/ctrl/worker/config.go +++ b/svc/ctrl/worker/config.go @@ -212,10 +212,6 @@ type Config struct { // Overrides default sentinel image with custom build or registry. SentinelImage string `toml:"sentinel_image" config:"default=ghcr.io/unkeyed/unkey:local"` - // AvailableRegions is a list of available regions for deployments. - // typically in the format "region.provider", ie "us-east-1.aws", "local.dev" - AvailableRegions []string `toml:"available_regions"` - // CnameDomain is the base domain for custom domain CNAME targets. // Each custom domain gets a unique subdomain like "{random}.{CnameDomain}". CnameDomain string `toml:"cname_domain" config:"required,nonempty"` diff --git a/svc/ctrl/worker/deploy/deploy_handler.go b/svc/ctrl/worker/deploy/deploy_handler.go index dccd6fe319c..0360ce1e38f 100644 --- a/svc/ctrl/worker/deploy/deploy_handler.go +++ b/svc/ctrl/worker/deploy/deploy_handler.go @@ -375,7 +375,7 @@ func (w *Workflow) buildImage(ctx restate.ObjectContext, req *hydrav1.DeployRequ // // Region selection uses the environment's runtime settings: if a region config is // present, only those regions are used with their configured replica counts; -// otherwise all of [Workflow.availableRegions] are used with 1 replica each. +// otherwise the deployment fails with a terminal error. // // createTopologies also registers compensations for every inserted // topology. Compensation deletes by deployment, region, and version so retries @@ -386,41 +386,32 @@ func (w *Workflow) createTopologies( workspace db.Workspace, deployment db.Deployment, ) ([]db.InsertDeploymentTopologyParams, error) { - // Read region config from runtime settings to determine per-region replica counts. - // If regionConfig is empty, deploy to all available regions with 1 replica each (default). - // If regionConfig has entries, only deploy to those regions with the specified counts. - regionConfig := map[string]int{} - runtimeSettings, err := restate.Run(ctx, func(runCtx restate.RunContext) (db.FindAppRuntimeSettingsByAppAndEnvRow, error) { - return db.Query.FindAppRuntimeSettingsByAppAndEnv(runCtx, w.db.RO(), db.FindAppRuntimeSettingsByAppAndEnvParams{ + // Read regional settings to determine per-region replica counts. + // If no regional settings exist, fail with a terminal error. + regionalSettings, err := restate.Run(ctx, func(runCtx restate.RunContext) ([]db.FindAppRegionalSettingsByAppAndEnvRow, error) { + return db.Query.FindAppRegionalSettingsByAppAndEnv(runCtx, w.db.RO(), db.FindAppRegionalSettingsByAppAndEnvParams{ AppID: deployment.AppID, EnvironmentID: deployment.EnvironmentID, }) - }, restate.WithName("find runtime settings for region config")) + }, restate.WithName("find regional settings")) if err != nil { return nil, fault.Wrap( - fmt.Errorf("failed to find runtime settings for environment %s: %w", deployment.EnvironmentID, err), + fmt.Errorf("failed to find regional settings for environment %s: %w", deployment.EnvironmentID, err), fault.Public("Failed to read from database. Please try again."), ) } - if len(runtimeSettings.AppRuntimeSetting.RegionConfig) > 0 { - for region, count := range runtimeSettings.AppRuntimeSetting.RegionConfig { - regionConfig[region] = count - } - } - var regions []string - if len(regionConfig) == 0 { - regions = w.availableRegions - } else { - for r := range regionConfig { - regions = append(regions, r) - } + if len(regionalSettings) == 0 { + return nil, fault.Wrap( + restate.TerminalError(fmt.Errorf("no regions configured for app %s in environment %s", deployment.AppID, deployment.EnvironmentID), 400), + fault.Public("No regions configured. Please configure at least one region before deploying."), + ) } - topologies := make([]db.InsertDeploymentTopologyParams, 0, len(regions)) + topologies := make([]db.InsertDeploymentTopologyParams, 0, len(regionalSettings)) - for _, region := range regions { - versionResp, err := hydrav1.NewVersioningServiceClient(ctx, region).NextVersion().Request(&hydrav1.NextVersionRequest{}) + for _, rs := range regionalSettings { + versionResp, err := hydrav1.NewVersioningServiceClient(ctx, rs.RegionName).NextVersion().Request(&hydrav1.NextVersionRequest{}) if err != nil { return nil, fault.Wrap( fmt.Errorf("failed to get next version: %w", err), @@ -428,17 +419,15 @@ func (w *Workflow) createTopologies( ) } - replicas := int32(1) - if count, ok := regionConfig[region]; ok { - replicas = int32(count) - } + replicas := rs.Replicas topologies = append(topologies, db.InsertDeploymentTopologyParams{ WorkspaceID: workspace.ID, DeploymentID: deployment.ID, - Region: region, + Region: rs.RegionName, + RegionID: rs.RegionID, DesiredReplicas: replicas, - DesiredStatus: db.DeploymentTopologyDesiredStatusStarting, + DesiredStatus: db.DeploymentTopologyDesiredStatusRunning, Version: versionResp.GetVersion(), CreatedAt: time.Now().UnixMilli(), }) diff --git a/svc/ctrl/worker/deploy/service.go b/svc/ctrl/worker/deploy/service.go index 2f662406702..7d08be4c1a1 100644 --- a/svc/ctrl/worker/deploy/service.go +++ b/svc/ctrl/worker/deploy/service.go @@ -42,11 +42,11 @@ type Workflow struct { hydrav1.UnimplementedDeployServiceServer db db.Database - defaultDomain string - vault vault.VaultServiceClient - sentinelImage string - availableRegions []string - github githubclient.GitHubClient + defaultDomain string + vault vault.VaultServiceClient + sentinelImage string + + github githubclient.GitHubClient // Build dependencies depotConfig DepotConfig @@ -72,9 +72,6 @@ type Config struct { // SentinelImage is the Docker image used for sentinel containers. SentinelImage string - // AvailableRegions is the list of available regions for deployments. - AvailableRegions []string - // GitHub provides access to GitHub API for downloading tarballs. GitHub githubclient.GitHubClient @@ -103,12 +100,12 @@ func New(cfg Config) *Workflow { defaultDomain: cfg.DefaultDomain, vault: cfg.Vault, sentinelImage: cfg.SentinelImage, - availableRegions: cfg.AvailableRegions, - github: cfg.GitHub, - depotConfig: cfg.DepotConfig, - registryConfig: cfg.RegistryConfig, - buildPlatform: cfg.BuildPlatform, - clickhouse: cfg.Clickhouse, - allowUnauthenticatedDeployments: cfg.AllowUnauthenticatedDeployments, + + github: cfg.GitHub, + depotConfig: cfg.DepotConfig, + registryConfig: cfg.RegistryConfig, + buildPlatform: cfg.BuildPlatform, + clickhouse: cfg.Clickhouse, + allowUnauthenticatedDeployments: cfg.AllowUnauthenticatedDeployments, } } diff --git a/svc/ctrl/worker/deployment/deployment_state.go b/svc/ctrl/worker/deployment/deployment_state.go index bf060feeb8f..1a5c8278971 100644 --- a/svc/ctrl/worker/deployment/deployment_state.go +++ b/svc/ctrl/worker/deployment/deployment_state.go @@ -81,7 +81,7 @@ func (v *VirtualObject) ChangeDesiredState(ctx restate.ObjectContext, req *hydra switch req.GetState() { case hydrav1.DeploymentDesiredState_DEPLOYMENT_DESIRED_STATE_RUNNING: desiredState = db.DeploymentsDesiredStateRunning - topologyDesiredStatus = db.DeploymentTopologyDesiredStatusStarted + topologyDesiredStatus = db.DeploymentTopologyDesiredStatusRunning case hydrav1.DeploymentDesiredState_DEPLOYMENT_DESIRED_STATE_STANDBY: desiredState = db.DeploymentsDesiredStateStandby topologyDesiredStatus = db.DeploymentTopologyDesiredStatusStopped diff --git a/svc/ctrl/worker/run.go b/svc/ctrl/worker/run.go index 32f991089ba..b2d525cb372 100644 --- a/svc/ctrl/worker/run.go +++ b/svc/ctrl/worker/run.go @@ -161,11 +161,11 @@ func Run(ctx context.Context, cfg Config) error { } restateSrv.Bind(hydrav1.NewDeployServiceServer(deploy.New(deploy.Config{ - DB: database, - DefaultDomain: cfg.DefaultDomain, - Vault: vaultClient, - SentinelImage: cfg.SentinelImage, - AvailableRegions: cfg.AvailableRegions, + DB: database, + DefaultDomain: cfg.DefaultDomain, + Vault: vaultClient, + SentinelImage: cfg.SentinelImage, + GitHub: ghClient, RegistryConfig: deploy.RegistryConfig(cfg.GetRegistryConfig()), BuildPlatform: deploy.BuildPlatform(buildPlatform), diff --git a/svc/frontline/config.go b/svc/frontline/config.go index 05b5365474f..85f67f7a063 100644 --- a/svc/frontline/config.go +++ b/svc/frontline/config.go @@ -30,6 +30,10 @@ type Config struct { // Serves general traffic over HTTPS by default. HttpPort int `toml:"http_port" config:"default=7443,min=1,max=65535"` + // Platform identifies the cloud provider + // ie: aws, gcp, local + Platform string `toml:"platform" config:"required"` + // Region identifies the geographic region where this node is deployed. // Used for observability, latency optimization, and cross-region routing. Region string `toml:"region" config:"required"` diff --git a/svc/frontline/routes/proxy/handler.go b/svc/frontline/routes/proxy/handler.go index fc8d67be3e2..8450d380ec2 100644 --- a/svc/frontline/routes/proxy/handler.go +++ b/svc/frontline/routes/proxy/handler.go @@ -48,5 +48,5 @@ func (h *Handler) Handle(ctx context.Context, sess *zen.Session) error { return h.ProxyService.ForwardToSentinel(ctx, sess, decision.LocalSentinel, decision.DeploymentID) } - return h.ProxyService.ForwardToRegion(ctx, sess, decision.NearestNLBRegion) + return h.ProxyService.ForwardToRegion(ctx, sess, decision.NearestNLBRegionPlatform) } diff --git a/svc/frontline/run.go b/svc/frontline/run.go index c661f70aec7..295704234e2 100644 --- a/svc/frontline/run.go +++ b/svc/frontline/run.go @@ -212,6 +212,7 @@ func Run(ctx context.Context, cfg Config) error { // Initialize router service routerSvc, err := router.New(router.Config{ + Platform: cfg.Platform, Region: cfg.Region, DB: db, FrontlineRouteCache: cache.FrontlineRoutes, @@ -225,6 +226,7 @@ func Run(ctx context.Context, cfg Config) error { // nolint:exhaustruct proxySvc, err := proxy.New(proxy.Config{ InstanceID: cfg.InstanceID, + Platform: cfg.Platform, Region: cfg.Region, ApexDomain: cfg.ApexDomain, Clock: clk, diff --git a/svc/frontline/services/proxy/director.go b/svc/frontline/services/proxy/director.go index 85c1ccfc8f0..73b416ec136 100644 --- a/svc/frontline/services/proxy/director.go +++ b/svc/frontline/services/proxy/director.go @@ -38,6 +38,7 @@ func (s *service) makeSentinelDirector(sess *zen.Session, deploymentID string, s func (s *service) makeRegionDirector(sess *zen.Session, startTime time.Time) func(*http.Request) { return func(req *http.Request) { req.Header.Set(HeaderFrontlineID, s.instanceID) + req.Header.Set(HeaderPlatform, s.platform) req.Header.Set(HeaderRegion, s.region) req.Header.Set(HeaderRequestID, sess.RequestID()) diff --git a/svc/frontline/services/proxy/forward.go b/svc/frontline/services/proxy/forward.go index 6bb0cf8b306..a161a98a208 100644 --- a/svc/frontline/services/proxy/forward.go +++ b/svc/frontline/services/proxy/forward.go @@ -29,6 +29,7 @@ type forwardConfig struct { func (s *service) forward(sess *zen.Session, cfg forwardConfig) error { sess.ResponseWriter().Header().Set(HeaderFrontlineID, s.instanceID) + sess.ResponseWriter().Header().Set(HeaderPlatform, s.platform) sess.ResponseWriter().Header().Set(HeaderRegion, s.region) sess.ResponseWriter().Header().Set(HeaderRequestID, sess.RequestID()) diff --git a/svc/frontline/services/proxy/headers.go b/svc/frontline/services/proxy/headers.go index 5add2b85100..246c22a572d 100644 --- a/svc/frontline/services/proxy/headers.go +++ b/svc/frontline/services/proxy/headers.go @@ -5,6 +5,7 @@ const ( // Headers set on BOTH response (to client) AND request (to downstream service) // These identify which frontline processed the request HeaderFrontlineID = "X-Unkey-Frontline-Id" // ID of the frontline instance + HeaderPlatform = "X-Unkey-Platform" // Platform of the frontline instance HeaderRegion = "X-Unkey-Region" // Region of the frontline instance HeaderRequestID = "X-Unkey-Request-Id" // Request ID for tracing diff --git a/svc/frontline/services/proxy/interface.go b/svc/frontline/services/proxy/interface.go index a2b32527e8e..0546a2652ac 100644 --- a/svc/frontline/services/proxy/interface.go +++ b/svc/frontline/services/proxy/interface.go @@ -21,7 +21,7 @@ type Service interface { // ForwardToRegion forwards a request to a remote region (HTTPS) // Keeps the original hostname so the remote frontline can do TLS termination and routing // Request start time is retrieved from context - ForwardToRegion(ctx context.Context, sess *zen.Session, targetRegion string) error + ForwardToRegion(ctx context.Context, sess *zen.Session, targetRegionPlatform string) error } // Config holds configuration for the proxy service. @@ -29,6 +29,7 @@ type Config struct { // InstanceID is the current frontline instance ID InstanceID string + Platform string // Region is the current frontline region Region string diff --git a/svc/frontline/services/proxy/service.go b/svc/frontline/services/proxy/service.go index b829d541744..c8db9bee5b1 100644 --- a/svc/frontline/services/proxy/service.go +++ b/svc/frontline/services/proxy/service.go @@ -22,6 +22,7 @@ import ( type service struct { instanceID string + platform string region string apexDomain string clock clock.Clock @@ -99,6 +100,7 @@ func New(cfg Config) (*service, error) { return &service{ instanceID: cfg.InstanceID, + platform: cfg.Platform, region: cfg.Region, apexDomain: cfg.ApexDomain, clock: cfg.Clock, @@ -129,7 +131,7 @@ func (s *service) ForwardToSentinel(ctx context.Context, sess *zen.Session, sent }) } -func (s *service) ForwardToRegion(ctx context.Context, sess *zen.Session, targetRegion string) error { +func (s *service) ForwardToRegion(ctx context.Context, sess *zen.Session, targetRegionPlatform string) error { startTime, _ := RequestStartTimeFromContext(ctx) if hopCountStr := sess.Request().Header.Get(HeaderFrontlineHops); hopCountStr != "" { @@ -148,7 +150,7 @@ func (s *service) ForwardToRegion(ctx context.Context, sess *zen.Session, target } } - targetURL, err := url.Parse(fmt.Sprintf("https://frontline.%s.%s", targetRegion, s.apexDomain)) + targetURL, err := url.Parse(fmt.Sprintf("https://frontline.%s.%s", targetRegionPlatform, s.apexDomain)) if err != nil { return fault.Wrap(err, fault.Code(codes.Frontline.Internal.InternalServerError.URN()), diff --git a/svc/frontline/services/router/interface.go b/svc/frontline/services/router/interface.go index f0ddf504ecd..c55d0712b77 100644 --- a/svc/frontline/services/router/interface.go +++ b/svc/frontline/services/router/interface.go @@ -11,8 +11,9 @@ type RouteDecision struct { // LocalSentinel is set if there's a healthy sentinel in the local region LocalSentinel *db.Sentinel - // NearestNLBRegion is set if we need to forward to another region's NLB - NearestNLBRegion string + // NearestNLBRegionPlatform is set if we need to forward to another region's NLB + // The format must be dns compatible and is typically "." + NearestNLBRegionPlatform string // DeploymentID to pass in X-Unkey-Deployment-Id header DeploymentID string @@ -24,6 +25,7 @@ type Service interface { } type Config struct { + Platform string Region string DB db.Database FrontlineRouteCache cache.Cache[string, db.FrontlineRoute] diff --git a/svc/frontline/services/router/service.go b/svc/frontline/services/router/service.go index e2c90aaba67..02eeaecf1fd 100644 --- a/svc/frontline/services/router/service.go +++ b/svc/frontline/services/router/service.go @@ -2,6 +2,7 @@ package router import ( "context" + "fmt" internalCaches "github.com/unkeyed/unkey/internal/services/caches" "github.com/unkeyed/unkey/pkg/cache" @@ -41,6 +42,7 @@ var regionProximity = map[string][]string{ } type service struct { + platform string region string db db.Database frontlineRouteCache cache.Cache[string, db.FrontlineRoute] @@ -51,6 +53,7 @@ var _ Service = (*service)(nil) func New(cfg Config) (*service, error) { return &service{ + platform: cfg.Platform, region: cfg.Region, db: cfg.DB, frontlineRouteCache: cfg.FrontlineRouteCache, @@ -95,9 +98,9 @@ func (s *service) LookupByHostname(ctx context.Context, hostname string) (*db.Fr func (s *service) SelectSentinel(route *db.FrontlineRoute, sentinels []db.Sentinel) (*RouteDecision, error) { decision := &RouteDecision{ - DeploymentID: route.DeploymentID, - LocalSentinel: nil, - NearestNLBRegion: "", + DeploymentID: route.DeploymentID, + LocalSentinel: nil, + NearestNLBRegionPlatform: "", } healthyByRegion := make(map[string]*db.Sentinel) @@ -123,16 +126,19 @@ func (s *service) SelectSentinel(route *db.FrontlineRoute, sentinels []db.Sentin return decision, nil } - nearestRegion := s.findNearestRegion(healthyByRegion) + nearestRegion := s.findNearestRegionPlatform(healthyByRegion) if nearestRegion != "" { - decision.NearestNLBRegion = nearestRegion + decision.NearestNLBRegionPlatform = nearestRegion } return decision, nil } -func (s *service) findNearestRegion(healthyByRegion map[string]*db.Sentinel) string { - proximityList, exists := regionProximity[s.region] +func (s *service) findNearestRegionPlatform(healthyByRegion map[string]*db.Sentinel) string { + + self := fmt.Sprintf("%s.%s", s.region, s.platform) + + proximityList, exists := regionProximity[self] if exists { for _, region := range proximityList { if _, ok := healthyByRegion[region]; ok { diff --git a/svc/krane/BUILD.bazel b/svc/krane/BUILD.bazel index 8b197c4f376..fe58e784e59 100644 --- a/svc/krane/BUILD.bazel +++ b/svc/krane/BUILD.bazel @@ -10,6 +10,7 @@ go_library( importpath = "github.com/unkeyed/unkey/svc/krane", visibility = ["//visibility:public"], deps = [ + "//gen/proto/ctrl/v1:ctrl", "//gen/proto/krane/v1/kranev1connect", "//gen/proto/vault/v1/vaultv1connect", "//gen/rpc/vault", @@ -18,6 +19,7 @@ go_library( "//pkg/logger", "//pkg/otel", "//pkg/prometheus", + "//pkg/repeat", "//pkg/rpc/interceptor", "//pkg/runner", "//pkg/version", diff --git a/svc/krane/config.go b/svc/krane/config.go index ec3b20c01f9..fb4a86e0828 100644 --- a/svc/krane/config.go +++ b/svc/krane/config.go @@ -37,6 +37,9 @@ type Config struct { // Region identifies the geographic region where this node is deployed. Region string `toml:"region" config:"required,nonempty"` + // Platform identifies the infrastructure provider (e.g. "aws", "gcp", "local"). + Platform string `toml:"platform" config:"required,nonempty"` + // RPCPort is the TCP port for the gRPC server. RPCPort int `toml:"rpc_port" config:"default=8070,min=1,max=65535"` diff --git a/svc/krane/internal/sentinel/apply.go b/svc/krane/internal/sentinel/apply.go index bd8b081658c..11ca456b39b 100644 --- a/svc/krane/internal/sentinel/apply.go +++ b/svc/krane/internal/sentinel/apply.go @@ -130,6 +130,7 @@ func (c *Controller) ensureSentinelExists(ctx context.Context, sentinel *ctrlv1. SentinelID: sentinel.GetSentinelId(), WorkspaceID: sentinel.GetWorkspaceId(), EnvironmentID: sentinel.GetEnvironmentId(), + Platform: c.platform, Region: c.region, HttpPort: SentinelPort, Database: config.DatabaseConfig{ @@ -169,6 +170,7 @@ func (c *Controller) ensureSentinelExists(ctx context.Context, sentinel *ctrlv1. Labels: labels.New(). WorkspaceID(sentinel.GetWorkspaceId()). ProjectID(sentinel.GetProjectId()). + AppID(sentinel.GetAppId()). EnvironmentID(sentinel.GetEnvironmentId()). SentinelID(sentinel.GetSentinelId()). ComponentSentinel(). @@ -319,6 +321,7 @@ func (c *Controller) ensureServiceExists(ctx context.Context, sentinel *ctrlv1.A Labels: labels.New(). WorkspaceID(sentinel.GetWorkspaceId()). ProjectID(sentinel.GetProjectId()). + AppID(sentinel.GetAppId()). EnvironmentID(sentinel.GetEnvironmentId()). SentinelID(sentinel.GetSentinelId()). ManagedByKrane(). @@ -388,6 +391,7 @@ func (c *Controller) ensurePDBExists(ctx context.Context, sentinel *ctrlv1.Apply Labels: labels.New(). WorkspaceID(sentinel.GetWorkspaceId()). ProjectID(sentinel.GetProjectId()). + AppID(sentinel.GetAppId()). EnvironmentID(sentinel.GetEnvironmentId()). SentinelID(sentinel.GetSentinelId()). ManagedByKrane(). @@ -440,6 +444,7 @@ func (c *Controller) ensureGossipServiceExists(ctx context.Context, sentinel *ct Labels: labels.New(). WorkspaceID(sentinel.GetWorkspaceId()). ProjectID(sentinel.GetProjectId()). + AppID(sentinel.GetAppId()). EnvironmentID(sentinel.GetEnvironmentId()). SentinelID(sentinel.GetSentinelId()). ComponentGossipLAN(), @@ -495,6 +500,7 @@ func (c *Controller) ensureGossipCiliumPolicyExists(ctx context.Context, sentine "labels": labels.New(). WorkspaceID(sentinel.GetWorkspaceId()). ProjectID(sentinel.GetProjectId()). + AppID(sentinel.GetAppId()). EnvironmentID(sentinel.GetEnvironmentId()). SentinelID(sentinel.GetSentinelId()). ComponentGossipLAN(), diff --git a/svc/krane/internal/sentinel/controller.go b/svc/krane/internal/sentinel/controller.go index 93b126ad6ce..ab93f279985 100644 --- a/svc/krane/internal/sentinel/controller.go +++ b/svc/krane/internal/sentinel/controller.go @@ -26,6 +26,7 @@ type Controller struct { done chan struct{} stopOnce sync.Once region string + platform string versionLastSeen uint64 } @@ -33,6 +34,7 @@ type Controller struct { type Config struct { Cluster ctrl.ClusterServiceClient Region string + Platform string ClientSet kubernetes.Interface DynamicClient dynamic.Interface } @@ -46,6 +48,7 @@ func New(cfg Config) *Controller { cb: circuitbreaker.New[any]("sentinel_state_update"), done: make(chan struct{}), region: cfg.Region, + platform: cfg.Platform, versionLastSeen: 0, stopOnce: sync.Once{}, } diff --git a/svc/krane/internal/testutil/mock_cluster_client.go b/svc/krane/internal/testutil/mock_cluster_client.go index f1db2690901..787e914de17 100644 --- a/svc/krane/internal/testutil/mock_cluster_client.go +++ b/svc/krane/internal/testutil/mock_cluster_client.go @@ -26,6 +26,7 @@ type MockClusterClient struct { GetDesiredDeploymentStateFunc func(context.Context, *ctrlv1.GetDesiredDeploymentStateRequest) (*ctrlv1.DeploymentState, error) ReportDeploymentStatusFunc func(context.Context, *ctrlv1.ReportDeploymentStatusRequest) (*ctrlv1.ReportDeploymentStatusResponse, error) GetDesiredCiliumNetworkPolicyStateFunc func(context.Context, *ctrlv1.GetDesiredCiliumNetworkPolicyStateRequest) (*ctrlv1.CiliumNetworkPolicyState, error) + HeartbeatFunc func(context.Context, *ctrlv1.HeartbeatRequest) (*ctrlv1.HeartbeatResponse, error) ReportDeploymentStatusCalls []*ctrlv1.ReportDeploymentStatusRequest ReportSentinelStatusCalls []*ctrlv1.ReportSentinelStatusRequest } @@ -87,3 +88,10 @@ func (m *MockClusterClient) GetDesiredCiliumNetworkPolicyState(ctx context.Conte } return &ctrlv1.CiliumNetworkPolicyState{}, nil } + +func (m *MockClusterClient) Heartbeat(ctx context.Context, req *ctrlv1.HeartbeatRequest) (*ctrlv1.HeartbeatResponse, error) { + if m.HeartbeatFunc != nil { + return m.HeartbeatFunc(ctx, req) + } + return &ctrlv1.HeartbeatResponse{}, nil +} diff --git a/svc/krane/pkg/controlplane/client.go b/svc/krane/pkg/controlplane/client.go index f97b5458b1e..77b7cc0152f 100644 --- a/svc/krane/pkg/controlplane/client.go +++ b/svc/krane/pkg/controlplane/client.go @@ -28,6 +28,10 @@ type ClientConfig struct { // Region identifies the geographical region of this client instance. // This value is added as the X-Krane-Region header for proper request routing. Region string + + // Platform identifies the infrastructure provider (e.g. "aws", "local"). + // This value is added as the X-Krane-Platform header. + Platform string } // NewClient creates a new control plane client with the specified configuration. @@ -70,6 +74,6 @@ func NewClient(cfg ClientConfig) ctrl.ClusterServiceClient { Transport: transport, }, cfg.URL, - connect.WithInterceptors(connectInterceptor(cfg.Region, cfg.BearerToken)), + connect.WithInterceptors(connectInterceptor(cfg.Region, cfg.Platform, cfg.BearerToken)), )) } diff --git a/svc/krane/pkg/controlplane/interceptor.go b/svc/krane/pkg/controlplane/interceptor.go index 9a0738a4715..6cc9ffb9384 100644 --- a/svc/krane/pkg/controlplane/interceptor.go +++ b/svc/krane/pkg/controlplane/interceptor.go @@ -16,10 +16,11 @@ import ( // - X-Krane-Region header for routing // // This is the recommended way to create interceptors for control plane clients. -func connectInterceptor(region, bearer string) connect.Interceptor { +func connectInterceptor(region, platform, bearer string) connect.Interceptor { return &authInterceptor{ - region: region, - bearer: bearer, + region: region, + platform: platform, + bearer: bearer, } } @@ -28,8 +29,9 @@ func connectInterceptor(region, bearer string) connect.Interceptor { // It automatically adds authentication and routing metadata to all outgoing requests. // The interceptor is stateless and safe for concurrent use. type authInterceptor struct { - region string - bearer string + region string + platform string + bearer string } // WrapUnary intercepts unary RPC calls by adding required headers before forwarding @@ -122,10 +124,12 @@ func (s *streamingClientInterceptor) RequestHeader() http.Header { // // This method injects: // - X-Krane-Region: The client's geographical region +// - X-Krane-Platform: The infrastructure provider (e.g. "aws", "local") // - Authorization: Bearer token for authentication // // All headers use Set() to overwrite any existing values. func (i *authInterceptor) setHeaders(header http.Header) { header.Set("X-Krane-Region", i.region) + header.Set("X-Krane-Platform", i.platform) header.Set("Authorization", fmt.Sprintf("Bearer %s", i.bearer)) } diff --git a/svc/krane/run.go b/svc/krane/run.go index b5fdefd7dff..e44c8ec8175 100644 --- a/svc/krane/run.go +++ b/svc/krane/run.go @@ -10,12 +10,14 @@ import ( "time" "connectrpc.com/connect" + ctrlv1 "github.com/unkeyed/unkey/gen/proto/ctrl/v1" "github.com/unkeyed/unkey/gen/proto/krane/v1/kranev1connect" "github.com/unkeyed/unkey/gen/proto/vault/v1/vaultv1connect" "github.com/unkeyed/unkey/gen/rpc/vault" "github.com/unkeyed/unkey/pkg/logger" "github.com/unkeyed/unkey/pkg/otel" "github.com/unkeyed/unkey/pkg/prometheus" + "github.com/unkeyed/unkey/pkg/repeat" "github.com/unkeyed/unkey/pkg/rpc/interceptor" "github.com/unkeyed/unkey/pkg/runner" pkgversion "github.com/unkeyed/unkey/pkg/version" @@ -90,14 +92,11 @@ func Run(ctx context.Context, cfg Config) error { r.DeferCtx(shutdownGrafana) - defer r.Recover() - - r.DeferCtx(shutdownGrafana) - cluster := controlplane.NewClient(controlplane.ClientConfig{ URL: cfg.Control.URL, BearerToken: cfg.Control.Token, Region: cfg.Region, + Platform: cfg.Platform, }) inClusterConfig, err := rest.InClusterConfig() @@ -145,12 +144,24 @@ func Run(ctx context.Context, cfg Config) error { DynamicClient: dynamicClient, Cluster: cluster, Region: cfg.Region, + Platform: cfg.Platform, }) if err := sentinelCtrl.Start(ctx); err != nil { return fmt.Errorf("failed to start sentinel controller: %w", err) } r.Defer(sentinelCtrl.Stop) + // Start heartbeat loop to register this cluster with the control plane + stopHeartbeat := repeat.Every(30*time.Second, func() { + if _, err := cluster.Heartbeat(ctx, &ctrlv1.HeartbeatRequest{ + Region: cfg.Region, + Platform: cfg.Platform, + }); err != nil { + logger.Warn("heartbeat failed", "error", err) + } + }) + r.Defer(func() error { stopHeartbeat(); return nil }) + // Create vault client for secrets decryption var vaultClient vault.VaultServiceClient if cfg.Vault.URL != "" { diff --git a/svc/sentinel/config.go b/svc/sentinel/config.go index 1eb0f053c0a..adce2e45c66 100644 --- a/svc/sentinel/config.go +++ b/svc/sentinel/config.go @@ -1,9 +1,6 @@ package sentinel import ( - "fmt" - "slices" - "github.com/unkeyed/unkey/pkg/config" ) @@ -46,9 +43,12 @@ type Config struct { // handles all of them based on the deployment ID passed in each request. EnvironmentID string `toml:"environment_id" config:"required,nonempty"` - // Region is the geographic region identifier (e.g. "us-east-1.aws"). + // Platform identifies the underlying cloud platform this sentinel is running on. + Platform string `toml:"platform" config:"required,nonempty"` + + // Region is the geographic region identifier (e.g. "us-east-1"). // Included in structured logs and used for routing decisions. - Region string `toml:"region" config:"required,oneof=local.dev|us-east-1.aws|us-east-2.aws|us-west-1.aws|us-west-2.aws|eu-central-1.aws"` + Region string `toml:"region" config:"required,nonempty"` // HttpPort is the TCP port the sentinel server binds to. HttpPort int `toml:"http_port" config:"default=8080,min=1,max=65535"` @@ -75,18 +75,6 @@ type Config struct { // struct tags alone. It implements [config.Validator] so that [config.Load] // calls it automatically after tag-level validation. func (c *Config) Validate() error { - validRegions := []string{ - "local.dev", - "us-east-1.aws", - "us-east-2.aws", - "us-west-1.aws", - "us-west-2.aws", - "eu-central-1.aws", - } - - if !slices.Contains(validRegions, c.Region) { - return fmt.Errorf("invalid region: %s, must be one of %v", c.Region, validRegions) - } return nil } diff --git a/svc/sentinel/run.go b/svc/sentinel/run.go index da030393348..7cec3c50a9c 100644 --- a/svc/sentinel/run.go +++ b/svc/sentinel/run.go @@ -68,6 +68,7 @@ func Run(ctx context.Context, cfg Config) error { slog.String("sentinelID", cfg.SentinelID), slog.String("workspaceID", cfg.WorkspaceID), slog.String("environmentID", cfg.EnvironmentID), + slog.String("platform", cfg.Platform), slog.String("region", cfg.Region), slog.String("version", version.Version), )) diff --git a/web/apps/dashboard/.env.example b/web/apps/dashboard/.env.example index 4399362debd..491b5d451f5 100644 --- a/web/apps/dashboard/.env.example +++ b/web/apps/dashboard/.env.example @@ -29,7 +29,5 @@ VAULT_TOKEN=vault-test-token-123 CLICKHOUSE_URL= # Optional -AVAILABLE_REGIONS=us-east-1 - TRIGGER_API_KEY= NEXT_PUBLIC_TRIGGER_PUBLIC_KEY= diff --git a/web/apps/dashboard/app/(app)/[workspaceSlug]/projects/[projectId]/(overview)/deployments/[deploymentId]/network/unkey-flow/components/nodes/types.ts b/web/apps/dashboard/app/(app)/[workspaceSlug]/projects/[projectId]/(overview)/deployments/[deploymentId]/network/unkey-flow/components/nodes/types.ts index 19cf1a2cec1..ec5c0be6c0b 100644 --- a/web/apps/dashboard/app/(app)/[workspaceSlug]/projects/[projectId]/(overview)/deployments/[deploymentId]/network/unkey-flow/components/nodes/types.ts +++ b/web/apps/dashboard/app/(app)/[workspaceSlug]/projects/[projectId]/(overview)/deployments/[deploymentId]/network/unkey-flow/components/nodes/types.ts @@ -25,7 +25,7 @@ type OriginNode = BaseNode & { type SentinelNode = BaseNode & { metadata: { type: "sentinel"; - flagCode: "us" | "de" | "au" | "jp" | "in" | "br"; + flagCode: "us" | "de" | "au" | "jp" | "in" | "br" | "local"; instances: number; replicas: number; } & BaseMetrics; @@ -77,6 +77,7 @@ const REGION_INFO: Record = { jp: { name: "AP Northeast", location: "Tokyo" }, in: { name: "AP South", location: "Mumbai" }, br: { name: "SA East", location: "São Paulo" }, + local: { name: "Local", location: "Local" }, } as const; const DEFAULT_NODE_WIDTH = 230; diff --git a/web/apps/dashboard/app/(app)/[workspaceSlug]/projects/[projectId]/(overview)/settings/components/runtime-settings/instances.tsx b/web/apps/dashboard/app/(app)/[workspaceSlug]/projects/[projectId]/(overview)/settings/components/runtime-settings/instances.tsx index 1fa1d272a30..3043d9d0079 100644 --- a/web/apps/dashboard/app/(app)/[workspaceSlug]/projects/[projectId]/(overview)/settings/components/runtime-settings/instances.tsx +++ b/web/apps/dashboard/app/(app)/[workspaceSlug]/projects/[projectId]/(overview)/settings/components/runtime-settings/instances.tsx @@ -21,10 +21,10 @@ type InstancesFormValues = z.infer; export const Instances = () => { const { settings, autoSave } = useEnvironmentSettings(); - const { environmentId, regionConfig } = settings; + const { environmentId, regions } = settings; - const selectedRegions = Object.keys(regionConfig); - const defaultInstances = Object.values(regionConfig)[0] ?? 1; + const selectedRegions = regions.map((r) => r.name); + const defaultInstances = regions.at(0)?.replicas ?? 1; const { handleSubmit, @@ -46,16 +46,14 @@ export const Instances = () => { const onSubmit = async (values: InstancesFormValues) => { collection.environmentSettings.update(environmentId, (draft) => { - const updated: Record = {}; - for (const region of Object.keys(draft.regionConfig)) { - updated[region] = values.instances; + for (const region of draft.regions) { + region.replicas = values.instances; } - draft.regionConfig = updated; }); }; const hasChanges = currentInstances !== defaultInstances; - const hasRegions = Object.keys(regionConfig).length > 0; + const hasRegions = regions.length > 0; const saveState = resolveSaveState([ [isSubmitting, { status: "saving" }], diff --git a/web/apps/dashboard/app/(app)/[workspaceSlug]/projects/[projectId]/(overview)/settings/components/runtime-settings/regions.tsx b/web/apps/dashboard/app/(app)/[workspaceSlug]/projects/[projectId]/(overview)/settings/components/runtime-settings/regions.tsx index 7bd0b3547b9..85b907addf5 100644 --- a/web/apps/dashboard/app/(app)/[workspaceSlug]/projects/[projectId]/(overview)/settings/components/runtime-settings/regions.tsx +++ b/web/apps/dashboard/app/(app)/[workspaceSlug]/projects/[projectId]/(overview)/settings/components/runtime-settings/regions.tsx @@ -20,10 +20,12 @@ const regionsSchema = z.object({ type RegionsFormValues = z.infer; +type AvailableRegion = { id: string; name: string }; + export const Regions = () => { const { settings, autoSave } = useEnvironmentSettings(); - const { environmentId, regionConfig } = settings; - const defaultRegions = Object.keys(regionConfig); + const { environmentId, regions } = settings; + const defaultRegionNames = regions.map((r) => r.name); const { data: availableRegions } = trpc.deploy.environmentSettings.getAvailableRegions.useQuery( undefined, @@ -33,7 +35,7 @@ export const Regions = () => { return ( @@ -42,14 +44,14 @@ export const Regions = () => { type RegionsFormProps = { environmentId: string; - defaultRegions: string[]; - availableRegions: string[]; + defaultRegionNames: string[]; + availableRegions: AvailableRegion[]; autoSave?: boolean; }; const RegionsForm: React.FC = ({ environmentId, - defaultRegions, + defaultRegionNames, availableRegions, autoSave, }) => { @@ -62,45 +64,48 @@ const RegionsForm: React.FC = ({ } = useForm({ resolver: zodResolver(regionsSchema), mode: "onChange", - defaultValues: { regions: defaultRegions }, + defaultValues: { regions: defaultRegionNames }, }); useEffect(() => { - reset({ regions: defaultRegions }); - }, [defaultRegions, reset]); + reset({ regions: defaultRegionNames }); + }, [defaultRegionNames, reset]); const currentRegions = useWatch({ control, name: "regions" }); - const unselectedRegions = availableRegions.filter((r) => !currentRegions.includes(r)); + const unselectedRegions = availableRegions.filter((r) => !currentRegions.includes(r.name)); const onSubmit = async (values: RegionsFormValues) => { collection.environmentSettings.update(environmentId, (draft) => { - const newConfig: Record = {}; - const defaultCount = Object.values(draft.regionConfig)[0] ?? 1; - for (const region of values.regions) { - newConfig[region] = draft.regionConfig[region] ?? defaultCount; - } - draft.regionConfig = newConfig; + const defaultReplicas = draft.regions.at(0)?.replicas ?? 1; + draft.regions = values.regions.map((name) => { + const existing = draft.regions.find((r) => r.name === name); + if (existing) { + return existing; + } + const available = availableRegions.find((r) => r.name === name); + return { id: available?.id ?? name, name, replicas: defaultReplicas }; + }); }); }; - const addRegion = (region: string) => { - if (region && !currentRegions.includes(region)) { - setValue("regions", [...currentRegions, region], { shouldValidate: true }); + const addRegion = (regionName: string) => { + if (regionName && !currentRegions.includes(regionName)) { + setValue("regions", [...currentRegions, regionName], { shouldValidate: true }); } }; - const removeRegion = (region: string) => { + const removeRegion = (regionName: string) => { setValue( "regions", - currentRegions.filter((r) => r !== region), + currentRegions.filter((r) => r !== regionName), { shouldValidate: true }, ); }; const hasChanges = - currentRegions.length !== defaultRegions.length || - currentRegions.some((r) => !defaultRegions.includes(r)); + currentRegions.length !== defaultRegionNames.length || + currentRegions.some((r) => !defaultRegionNames.includes(r)); const saveState = resolveSaveState([ [isSubmitting, { status: "saving" }], @@ -109,9 +114,9 @@ const RegionsForm: React.FC = ({ ]); const displayValue = - defaultRegions.length === 0 ? null : defaultRegions.length <= 2 ? ( + defaultRegionNames.length === 0 ? null : defaultRegionNames.length <= 2 ? ( - {defaultRegions.map((r, i) => ( + {defaultRegionNames.map((r, i) => ( {i > 0 && |} @@ -128,19 +133,19 @@ const RegionsForm: React.FC = ({ ) : ( - {defaultRegions.map((r) => ( + {defaultRegionNames.map((r) => ( ))} ); const comboboxOptions: ComboboxOption[] = unselectedRegions.map((region) => ({ - value: region, - searchValue: region, + value: region.name, + searchValue: region.name, label: (
- - {region} + + {region.name}
), })); diff --git a/web/apps/dashboard/app/(app)/[workspaceSlug]/projects/[projectId]/components/region-flag.tsx b/web/apps/dashboard/app/(app)/[workspaceSlug]/projects/[projectId]/components/region-flag.tsx index cca7e8193fe..a8642c10d76 100644 --- a/web/apps/dashboard/app/(app)/[workspaceSlug]/projects/[projectId]/components/region-flag.tsx +++ b/web/apps/dashboard/app/(app)/[workspaceSlug]/projects/[projectId]/components/region-flag.tsx @@ -1,9 +1,8 @@ import type { FlagCode } from "@/lib/trpc/routers/deploy/network/utils"; import { cn } from "@/lib/utils"; - +import BoringAvatar from "boring-avatars"; type RegionFlagSize = "xs" | "sm" | "md" | "lg"; type RegionFlagShape = "rounded" | "circle"; - type RegionFlagProps = { flagCode: FlagCode; size?: RegionFlagSize; @@ -47,7 +46,6 @@ export function RegionFlag({ }: RegionFlagProps) { const config = sizeConfig[size]; const hasExplicitPadding = config.padding !== ""; - return (
- {flagCode} + {flagCode === "local" ? ( + + ) : ( + {flagCode} + )}
); } diff --git a/web/apps/dashboard/app/(app)/[workspaceSlug]/projects/new/steps/onboarding-environment-provider.tsx b/web/apps/dashboard/app/(app)/[workspaceSlug]/projects/new/steps/onboarding-environment-provider.tsx index 04f09c6132f..bc1500c42bc 100644 --- a/web/apps/dashboard/app/(app)/[workspaceSlug]/projects/new/steps/onboarding-environment-provider.tsx +++ b/web/apps/dashboard/app/(app)/[workspaceSlug]/projects/new/steps/onboarding-environment-provider.tsx @@ -66,7 +66,7 @@ export const OnboardingEnvironmentSettingsProvider = ({ // Later they can change it in the settings. function useInitializeSettings( settings: EnvironmentSettings | undefined, - availableRegions: string[] | undefined, + availableRegions: { id: string; name: string }[] | undefined, isActive: boolean, ) { const hasInitializedRef = useRef(false); @@ -102,10 +102,8 @@ function useInitializeSettings( if (!draft.shutdownSignal) { draft.shutdownSignal = "SIGTERM"; } - if (Object.keys(draft.regionConfig).length === 0) { - for (const region of availableRegions) { - draft.regionConfig[region] = 1; - } + if (draft.regions.length === 0) { + draft.regions = availableRegions.map((r) => ({ id: r.id, name: r.name, replicas: 1 })); } }, ); diff --git a/web/apps/dashboard/gen/proto/ctrl/v1/cluster_pb.ts b/web/apps/dashboard/gen/proto/ctrl/v1/cluster_pb.ts index f3591e01979..cc70282a350 100644 --- a/web/apps/dashboard/gen/proto/ctrl/v1/cluster_pb.ts +++ b/web/apps/dashboard/gen/proto/ctrl/v1/cluster_pb.ts @@ -20,7 +20,7 @@ import type { Message } from "@bufbuild/protobuf"; * Describes the file ctrl/v1/cluster.proto. */ export const file_ctrl_v1_cluster: GenFile = /*@__PURE__*/ - fileDesc("ChVjdHJsL3YxL2NsdXN0ZXIucHJvdG8SB2N0cmwudjEiTgohV2F0Y2hDaWxpdW1OZXR3b3JrUG9saWNpZXNSZXF1ZXN0Eg4KBnJlZ2lvbhgBIAEoCRIZChF2ZXJzaW9uX2xhc3Rfc2VlbhgCIAEoBCJ1ChhBcHBseUNpbGl1bU5ldHdvcmtQb2xpY3kSFQoNazhzX25hbWVzcGFjZRgBIAEoCRIQCghrOHNfbmFtZRgCIAEoCRIgChhjaWxpdW1fbmV0d29ya19wb2xpY3lfaWQYAyABKAkSDgoGcG9saWN5GAQgASgMIkQKGURlbGV0ZUNpbGl1bU5ldHdvcmtQb2xpY3kSFQoNazhzX25hbWVzcGFjZRgBIAEoCRIQCghrOHNfbmFtZRgCIAEoCSKeAQoYQ2lsaXVtTmV0d29ya1BvbGljeVN0YXRlEg8KB3ZlcnNpb24YAyABKAQSMgoFYXBwbHkYASABKAsyIS5jdHJsLnYxLkFwcGx5Q2lsaXVtTmV0d29ya1BvbGljeUgAEjQKBmRlbGV0ZRgCIAEoCzIiLmN0cmwudjEuRGVsZXRlQ2lsaXVtTmV0d29ya1BvbGljeUgAQgcKBXN0YXRlIk0KKUdldERlc2lyZWRDaWxpdW1OZXR3b3JrUG9saWN5U3RhdGVSZXF1ZXN0EiAKGGNpbGl1bV9uZXR3b3JrX3BvbGljeV9pZBgBIAEoCSI1Ch5HZXREZXNpcmVkU2VudGluZWxTdGF0ZVJlcXVlc3QSEwoLc2VudGluZWxfaWQYASABKAkiOQogR2V0RGVzaXJlZERlcGxveW1lbnRTdGF0ZVJlcXVlc3QSFQoNZGVwbG95bWVudF9pZBgBIAEoCSK3BAodUmVwb3J0RGVwbG95bWVudFN0YXR1c1JlcXVlc3QSPwoGdXBkYXRlGAEgASgLMi0uY3RybC52MS5SZXBvcnREZXBsb3ltZW50U3RhdHVzUmVxdWVzdC5VcGRhdGVIABI/CgZkZWxldGUYAiABKAsyLS5jdHJsLnYxLlJlcG9ydERlcGxveW1lbnRTdGF0dXNSZXF1ZXN0LkRlbGV0ZUgAGu0CCgZVcGRhdGUSEAoIazhzX25hbWUYASABKAkSSQoJaW5zdGFuY2VzGAIgAygLMjYuY3RybC52MS5SZXBvcnREZXBsb3ltZW50U3RhdHVzUmVxdWVzdC5VcGRhdGUuSW5zdGFuY2UahQIKCEluc3RhbmNlEhAKCGs4c19uYW1lGAEgASgJEg8KB2FkZHJlc3MYAiABKAkSFgoOY3B1X21pbGxpY29yZXMYAyABKAMSEgoKbWVtb3J5X21pYhgEIAEoAxJNCgZzdGF0dXMYBSABKA4yPS5jdHJsLnYxLlJlcG9ydERlcGxveW1lbnRTdGF0dXNSZXF1ZXN0LlVwZGF0ZS5JbnN0YW5jZS5TdGF0dXMiWwoGU3RhdHVzEhYKElNUQVRVU19VTlNQRUNJRklFRBAAEhIKDlNUQVRVU19QRU5ESU5HEAESEgoOU1RBVFVTX1JVTk5JTkcQAhIRCg1TVEFUVVNfRkFJTEVEEAMaGgoGRGVsZXRlEhAKCGs4c19uYW1lGAEgASgJQggKBmNoYW5nZSIgCh5SZXBvcnREZXBsb3ltZW50U3RhdHVzUmVzcG9uc2UibAobUmVwb3J0U2VudGluZWxTdGF0dXNSZXF1ZXN0EhAKCGs4c19uYW1lGAEgASgJEhoKEmF2YWlsYWJsZV9yZXBsaWNhcxgCIAEoBRIfCgZoZWFsdGgYAyABKA4yDy5jdHJsLnYxLkhlYWx0aCIeChxSZXBvcnRTZW50aW5lbFN0YXR1c1Jlc3BvbnNlIkQKF1dhdGNoRGVwbG95bWVudHNSZXF1ZXN0Eg4KBnJlZ2lvbhgBIAEoCRIZChF2ZXJzaW9uX2xhc3Rfc2VlbhgCIAEoBCJCChVXYXRjaFNlbnRpbmVsc1JlcXVlc3QSDgoGcmVnaW9uGAEgASgJEhkKEXZlcnNpb25fbGFzdF9zZWVuGAIgASgEIn0KDVNlbnRpbmVsU3RhdGUSDwoHdmVyc2lvbhgDIAEoBBInCgVhcHBseRgBIAEoCzIWLmN0cmwudjEuQXBwbHlTZW50aW5lbEgAEikKBmRlbGV0ZRgCIAEoCzIXLmN0cmwudjEuRGVsZXRlU2VudGluZWxIAEIHCgVzdGF0ZSKDAQoPRGVwbG95bWVudFN0YXRlEg8KB3ZlcnNpb24YAyABKAQSKQoFYXBwbHkYASABKAsyGC5jdHJsLnYxLkFwcGx5RGVwbG95bWVudEgAEisKBmRlbGV0ZRgCIAEoCzIZLmN0cmwudjEuRGVsZXRlRGVwbG95bWVudEgAQgcKBXN0YXRlIsUBCg1BcHBseVNlbnRpbmVsEhAKCGs4c19uYW1lGAEgASgJEhQKDHdvcmtzcGFjZV9pZBgCIAEoCRISCgpwcm9qZWN0X2lkGAMgASgJEhYKDmVudmlyb25tZW50X2lkGAQgASgJEhMKC3NlbnRpbmVsX2lkGAUgASgJEg0KBWltYWdlGAYgASgJEhAKCHJlcGxpY2FzGAcgASgFEhYKDmNwdV9taWxsaWNvcmVzGAggASgDEhIKCm1lbW9yeV9taWIYCSABKAMiIgoORGVsZXRlU2VudGluZWwSEAoIazhzX25hbWUYASABKAkinwMKD0FwcGx5RGVwbG95bWVudBIVCg1rOHNfbmFtZXNwYWNlGAEgASgJEhAKCGs4c19uYW1lGAIgASgJEhQKDHdvcmtzcGFjZV9pZBgDIAEoCRISCgpwcm9qZWN0X2lkGAQgASgJEhYKDmVudmlyb25tZW50X2lkGAUgASgJEhUKDWRlcGxveW1lbnRfaWQYBiABKAkSDQoFaW1hZ2UYByABKAkSEAoIcmVwbGljYXMYCCABKAUSFgoOY3B1X21pbGxpY29yZXMYCSABKAMSEgoKbWVtb3J5X21pYhgKIAEoAxIVCghidWlsZF9pZBgLIAEoCUgAiAEBEicKH2VuY3J5cHRlZF9lbnZpcm9ubWVudF92YXJpYWJsZXMYDCABKAwSDwoHY29tbWFuZBgNIAMoCRIMCgRwb3J0GA4gASgFEhcKD3NodXRkb3duX3NpZ25hbBgPIAEoCRIYCgtoZWFsdGhjaGVjaxgRIAEoDEgBiAEBEg4KBmFwcF9pZBgSIAEoCUILCglfYnVpbGRfaWRCDgoMX2hlYWx0aGNoZWNrIjsKEERlbGV0ZURlcGxveW1lbnQSFQoNazhzX25hbWVzcGFjZRgBIAEoCRIQCghrOHNfbmFtZRgCIAEoCSpdCgZIZWFsdGgSFgoSSEVBTFRIX1VOU1BFQ0lGSUVEEAASEgoOSEVBTFRIX0hFQUxUSFkQARIUChBIRUFMVEhfVU5IRUFMVEhZEAISEQoNSEVBTFRIX1BBVVNFRBADMqgGCg5DbHVzdGVyU2VydmljZRJQChBXYXRjaERlcGxveW1lbnRzEiAuY3RybC52MS5XYXRjaERlcGxveW1lbnRzUmVxdWVzdBoYLmN0cmwudjEuRGVwbG95bWVudFN0YXRlMAESSgoOV2F0Y2hTZW50aW5lbHMSHi5jdHJsLnYxLldhdGNoU2VudGluZWxzUmVxdWVzdBoWLmN0cmwudjEuU2VudGluZWxTdGF0ZTABEloKF0dldERlc2lyZWRTZW50aW5lbFN0YXRlEicuY3RybC52MS5HZXREZXNpcmVkU2VudGluZWxTdGF0ZVJlcXVlc3QaFi5jdHJsLnYxLlNlbnRpbmVsU3RhdGUSYwoUUmVwb3J0U2VudGluZWxTdGF0dXMSJC5jdHJsLnYxLlJlcG9ydFNlbnRpbmVsU3RhdHVzUmVxdWVzdBolLmN0cmwudjEuUmVwb3J0U2VudGluZWxTdGF0dXNSZXNwb25zZRJgChlHZXREZXNpcmVkRGVwbG95bWVudFN0YXRlEikuY3RybC52MS5HZXREZXNpcmVkRGVwbG95bWVudFN0YXRlUmVxdWVzdBoYLmN0cmwudjEuRGVwbG95bWVudFN0YXRlEmkKFlJlcG9ydERlcGxveW1lbnRTdGF0dXMSJi5jdHJsLnYxLlJlcG9ydERlcGxveW1lbnRTdGF0dXNSZXF1ZXN0GicuY3RybC52MS5SZXBvcnREZXBsb3ltZW50U3RhdHVzUmVzcG9uc2USbQoaV2F0Y2hDaWxpdW1OZXR3b3JrUG9saWNpZXMSKi5jdHJsLnYxLldhdGNoQ2lsaXVtTmV0d29ya1BvbGljaWVzUmVxdWVzdBohLmN0cmwudjEuQ2lsaXVtTmV0d29ya1BvbGljeVN0YXRlMAESewoiR2V0RGVzaXJlZENpbGl1bU5ldHdvcmtQb2xpY3lTdGF0ZRIyLmN0cmwudjEuR2V0RGVzaXJlZENpbGl1bU5ldHdvcmtQb2xpY3lTdGF0ZVJlcXVlc3QaIS5jdHJsLnYxLkNpbGl1bU5ldHdvcmtQb2xpY3lTdGF0ZUKLAQoLY29tLmN0cmwudjFCDENsdXN0ZXJQcm90b1ABWjFnaXRodWIuY29tL3Vua2V5ZWQvdW5rZXkvZ2VuL3Byb3RvL2N0cmwvdjE7Y3RybHYxogIDQ1hYqgIHQ3RybC5WMcoCB0N0cmxcVjHiAhNDdHJsXFYxXEdQQk1ldGFkYXRh6gIIQ3RybDo6VjFiBnByb3RvMw"); + fileDesc("ChVjdHJsL3YxL2NsdXN0ZXIucHJvdG8SB2N0cmwudjEiTgohV2F0Y2hDaWxpdW1OZXR3b3JrUG9saWNpZXNSZXF1ZXN0Eg4KBnJlZ2lvbhgBIAEoCRIZChF2ZXJzaW9uX2xhc3Rfc2VlbhgCIAEoBCJ1ChhBcHBseUNpbGl1bU5ldHdvcmtQb2xpY3kSFQoNazhzX25hbWVzcGFjZRgBIAEoCRIQCghrOHNfbmFtZRgCIAEoCRIgChhjaWxpdW1fbmV0d29ya19wb2xpY3lfaWQYAyABKAkSDgoGcG9saWN5GAQgASgMIkQKGURlbGV0ZUNpbGl1bU5ldHdvcmtQb2xpY3kSFQoNazhzX25hbWVzcGFjZRgBIAEoCRIQCghrOHNfbmFtZRgCIAEoCSKeAQoYQ2lsaXVtTmV0d29ya1BvbGljeVN0YXRlEg8KB3ZlcnNpb24YAyABKAQSMgoFYXBwbHkYASABKAsyIS5jdHJsLnYxLkFwcGx5Q2lsaXVtTmV0d29ya1BvbGljeUgAEjQKBmRlbGV0ZRgCIAEoCzIiLmN0cmwudjEuRGVsZXRlQ2lsaXVtTmV0d29ya1BvbGljeUgAQgcKBXN0YXRlIk0KKUdldERlc2lyZWRDaWxpdW1OZXR3b3JrUG9saWN5U3RhdGVSZXF1ZXN0EiAKGGNpbGl1bV9uZXR3b3JrX3BvbGljeV9pZBgBIAEoCSI1Ch5HZXREZXNpcmVkU2VudGluZWxTdGF0ZVJlcXVlc3QSEwoLc2VudGluZWxfaWQYASABKAkiOQogR2V0RGVzaXJlZERlcGxveW1lbnRTdGF0ZVJlcXVlc3QSFQoNZGVwbG95bWVudF9pZBgBIAEoCSK3BAodUmVwb3J0RGVwbG95bWVudFN0YXR1c1JlcXVlc3QSPwoGdXBkYXRlGAEgASgLMi0uY3RybC52MS5SZXBvcnREZXBsb3ltZW50U3RhdHVzUmVxdWVzdC5VcGRhdGVIABI/CgZkZWxldGUYAiABKAsyLS5jdHJsLnYxLlJlcG9ydERlcGxveW1lbnRTdGF0dXNSZXF1ZXN0LkRlbGV0ZUgAGu0CCgZVcGRhdGUSEAoIazhzX25hbWUYASABKAkSSQoJaW5zdGFuY2VzGAIgAygLMjYuY3RybC52MS5SZXBvcnREZXBsb3ltZW50U3RhdHVzUmVxdWVzdC5VcGRhdGUuSW5zdGFuY2UahQIKCEluc3RhbmNlEhAKCGs4c19uYW1lGAEgASgJEg8KB2FkZHJlc3MYAiABKAkSFgoOY3B1X21pbGxpY29yZXMYAyABKAMSEgoKbWVtb3J5X21pYhgEIAEoAxJNCgZzdGF0dXMYBSABKA4yPS5jdHJsLnYxLlJlcG9ydERlcGxveW1lbnRTdGF0dXNSZXF1ZXN0LlVwZGF0ZS5JbnN0YW5jZS5TdGF0dXMiWwoGU3RhdHVzEhYKElNUQVRVU19VTlNQRUNJRklFRBAAEhIKDlNUQVRVU19QRU5ESU5HEAESEgoOU1RBVFVTX1JVTk5JTkcQAhIRCg1TVEFUVVNfRkFJTEVEEAMaGgoGRGVsZXRlEhAKCGs4c19uYW1lGAEgASgJQggKBmNoYW5nZSIgCh5SZXBvcnREZXBsb3ltZW50U3RhdHVzUmVzcG9uc2UibAobUmVwb3J0U2VudGluZWxTdGF0dXNSZXF1ZXN0EhAKCGs4c19uYW1lGAEgASgJEhoKEmF2YWlsYWJsZV9yZXBsaWNhcxgCIAEoBRIfCgZoZWFsdGgYAyABKA4yDy5jdHJsLnYxLkhlYWx0aCIeChxSZXBvcnRTZW50aW5lbFN0YXR1c1Jlc3BvbnNlIkQKF1dhdGNoRGVwbG95bWVudHNSZXF1ZXN0Eg4KBnJlZ2lvbhgBIAEoCRIZChF2ZXJzaW9uX2xhc3Rfc2VlbhgCIAEoBCJCChVXYXRjaFNlbnRpbmVsc1JlcXVlc3QSDgoGcmVnaW9uGAEgASgJEhkKEXZlcnNpb25fbGFzdF9zZWVuGAIgASgEIn0KDVNlbnRpbmVsU3RhdGUSDwoHdmVyc2lvbhgDIAEoBBInCgVhcHBseRgBIAEoCzIWLmN0cmwudjEuQXBwbHlTZW50aW5lbEgAEikKBmRlbGV0ZRgCIAEoCzIXLmN0cmwudjEuRGVsZXRlU2VudGluZWxIAEIHCgVzdGF0ZSKDAQoPRGVwbG95bWVudFN0YXRlEg8KB3ZlcnNpb24YAyABKAQSKQoFYXBwbHkYASABKAsyGC5jdHJsLnYxLkFwcGx5RGVwbG95bWVudEgAEisKBmRlbGV0ZRgCIAEoCzIZLmN0cmwudjEuRGVsZXRlRGVwbG95bWVudEgAQgcKBXN0YXRlItUBCg1BcHBseVNlbnRpbmVsEhAKCGs4c19uYW1lGAEgASgJEhQKDHdvcmtzcGFjZV9pZBgCIAEoCRISCgpwcm9qZWN0X2lkGAMgASgJEg4KBmFwcF9pZBgEIAEoCRIWCg5lbnZpcm9ubWVudF9pZBgFIAEoCRITCgtzZW50aW5lbF9pZBgGIAEoCRINCgVpbWFnZRgHIAEoCRIQCghyZXBsaWNhcxgIIAEoBRIWCg5jcHVfbWlsbGljb3JlcxgJIAEoAxISCgptZW1vcnlfbWliGAogASgDIiIKDkRlbGV0ZVNlbnRpbmVsEhAKCGs4c19uYW1lGAEgASgJIp8DCg9BcHBseURlcGxveW1lbnQSFQoNazhzX25hbWVzcGFjZRgBIAEoCRIQCghrOHNfbmFtZRgCIAEoCRIUCgx3b3Jrc3BhY2VfaWQYAyABKAkSEgoKcHJvamVjdF9pZBgEIAEoCRIWCg5lbnZpcm9ubWVudF9pZBgFIAEoCRIVCg1kZXBsb3ltZW50X2lkGAYgASgJEg0KBWltYWdlGAcgASgJEhAKCHJlcGxpY2FzGAggASgFEhYKDmNwdV9taWxsaWNvcmVzGAkgASgDEhIKCm1lbW9yeV9taWIYCiABKAMSFQoIYnVpbGRfaWQYCyABKAlIAIgBARInCh9lbmNyeXB0ZWRfZW52aXJvbm1lbnRfdmFyaWFibGVzGAwgASgMEg8KB2NvbW1hbmQYDSADKAkSDAoEcG9ydBgOIAEoBRIXCg9zaHV0ZG93bl9zaWduYWwYDyABKAkSGAoLaGVhbHRoY2hlY2sYESABKAxIAYgBARIOCgZhcHBfaWQYEiABKAlCCwoJX2J1aWxkX2lkQg4KDF9oZWFsdGhjaGVjayI7ChBEZWxldGVEZXBsb3ltZW50EhUKDWs4c19uYW1lc3BhY2UYASABKAkSEAoIazhzX25hbWUYAiABKAkiNAoQSGVhcnRiZWF0UmVxdWVzdBIOCgZyZWdpb24YASABKAkSEAoIcGxhdGZvcm0YAiABKAkiEwoRSGVhcnRiZWF0UmVzcG9uc2UqXQoGSGVhbHRoEhYKEkhFQUxUSF9VTlNQRUNJRklFRBAAEhIKDkhFQUxUSF9IRUFMVEhZEAESFAoQSEVBTFRIX1VOSEVBTFRIWRACEhEKDUhFQUxUSF9QQVVTRUQQAzLsBgoOQ2x1c3RlclNlcnZpY2USUAoQV2F0Y2hEZXBsb3ltZW50cxIgLmN0cmwudjEuV2F0Y2hEZXBsb3ltZW50c1JlcXVlc3QaGC5jdHJsLnYxLkRlcGxveW1lbnRTdGF0ZTABEkoKDldhdGNoU2VudGluZWxzEh4uY3RybC52MS5XYXRjaFNlbnRpbmVsc1JlcXVlc3QaFi5jdHJsLnYxLlNlbnRpbmVsU3RhdGUwARJaChdHZXREZXNpcmVkU2VudGluZWxTdGF0ZRInLmN0cmwudjEuR2V0RGVzaXJlZFNlbnRpbmVsU3RhdGVSZXF1ZXN0GhYuY3RybC52MS5TZW50aW5lbFN0YXRlEmMKFFJlcG9ydFNlbnRpbmVsU3RhdHVzEiQuY3RybC52MS5SZXBvcnRTZW50aW5lbFN0YXR1c1JlcXVlc3QaJS5jdHJsLnYxLlJlcG9ydFNlbnRpbmVsU3RhdHVzUmVzcG9uc2USYAoZR2V0RGVzaXJlZERlcGxveW1lbnRTdGF0ZRIpLmN0cmwudjEuR2V0RGVzaXJlZERlcGxveW1lbnRTdGF0ZVJlcXVlc3QaGC5jdHJsLnYxLkRlcGxveW1lbnRTdGF0ZRJpChZSZXBvcnREZXBsb3ltZW50U3RhdHVzEiYuY3RybC52MS5SZXBvcnREZXBsb3ltZW50U3RhdHVzUmVxdWVzdBonLmN0cmwudjEuUmVwb3J0RGVwbG95bWVudFN0YXR1c1Jlc3BvbnNlEm0KGldhdGNoQ2lsaXVtTmV0d29ya1BvbGljaWVzEiouY3RybC52MS5XYXRjaENpbGl1bU5ldHdvcmtQb2xpY2llc1JlcXVlc3QaIS5jdHJsLnYxLkNpbGl1bU5ldHdvcmtQb2xpY3lTdGF0ZTABEnsKIkdldERlc2lyZWRDaWxpdW1OZXR3b3JrUG9saWN5U3RhdGUSMi5jdHJsLnYxLkdldERlc2lyZWRDaWxpdW1OZXR3b3JrUG9saWN5U3RhdGVSZXF1ZXN0GiEuY3RybC52MS5DaWxpdW1OZXR3b3JrUG9saWN5U3RhdGUSQgoJSGVhcnRiZWF0EhkuY3RybC52MS5IZWFydGJlYXRSZXF1ZXN0GhouY3RybC52MS5IZWFydGJlYXRSZXNwb25zZUKLAQoLY29tLmN0cmwudjFCDENsdXN0ZXJQcm90b1ABWjFnaXRodWIuY29tL3Vua2V5ZWQvdW5rZXkvZ2VuL3Byb3RvL2N0cmwvdjE7Y3RybHYxogIDQ1hYqgIHQ3RybC5WMcoCB0N0cmxcVjHiAhNDdHJsXFYxXEdQQk1ldGFkYXRh6gIIQ3RybDo6VjFiBnByb3RvMw"); /** * @generated from message ctrl.v1.WatchCiliumNetworkPoliciesRequest @@ -575,37 +575,44 @@ export type ApplySentinel = Message<"ctrl.v1.ApplySentinel"> & { */ projectId: string; + /** + * app_id identifies the app within the project. + * + * @generated from field: string app_id = 4; + */ + appId: string; + /** * environment_id in which the sentinel should exist. * - * @generated from field: string environment_id = 4; + * @generated from field: string environment_id = 5; */ environmentId: string; /** * sentinel_id is the unique identifier for this sentinel globally * - * @generated from field: string sentinel_id = 5; + * @generated from field: string sentinel_id = 6; */ sentinelId: string; /** - * @generated from field: string image = 6; + * @generated from field: string image = 7; */ image: string; /** - * @generated from field: int32 replicas = 7; + * @generated from field: int32 replicas = 8; */ replicas: number; /** - * @generated from field: int64 cpu_millicores = 8; + * @generated from field: int64 cpu_millicores = 9; */ cpuMillicores: bigint; /** - * @generated from field: int64 memory_mib = 9; + * @generated from field: int64 memory_mib = 10; */ memoryMib: bigint; }; @@ -812,6 +819,49 @@ export type DeleteDeployment = Message<"ctrl.v1.DeleteDeployment"> & { export const DeleteDeploymentSchema: GenMessage = /*@__PURE__*/ messageDesc(file_ctrl_v1_cluster, 18); +/** + * HeartbeatRequest is sent periodically by krane agents to register their + * presence. The control plane uses this to populate regions and + * clusters tables. + * + * @generated from message ctrl.v1.HeartbeatRequest + */ +export type HeartbeatRequest = Message<"ctrl.v1.HeartbeatRequest"> & { + /** + * region is the geographic region name (e.g. "us-east-1"). + * + * @generated from field: string region = 1; + */ + region: string; + + /** + * platform is the infrastructure provider (e.g. "aws", "gcp", "local"). + * + * @generated from field: string platform = 2; + */ + platform: string; +}; + +/** + * Describes the message ctrl.v1.HeartbeatRequest. + * Use `create(HeartbeatRequestSchema)` to create a new message. + */ +export const HeartbeatRequestSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_ctrl_v1_cluster, 19); + +/** + * @generated from message ctrl.v1.HeartbeatResponse + */ +export type HeartbeatResponse = Message<"ctrl.v1.HeartbeatResponse"> & { +}; + +/** + * Describes the message ctrl.v1.HeartbeatResponse. + * Use `create(HeartbeatResponseSchema)` to create a new message. + */ +export const HeartbeatResponseSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_ctrl_v1_cluster, 20); + /** * Health represents the health state of a resource (sentinel, deployment instance, etc.) * @@ -949,6 +999,18 @@ export const ClusterService: GenService<{ input: typeof GetDesiredCiliumNetworkPolicyStateRequestSchema; output: typeof CiliumNetworkPolicyStateSchema; }, + /** + * Heartbeat is called periodically by krane agents to register their cluster + * and region with the control plane. This populates the regions and + * clusters tables, making regions dynamically discoverable. + * + * @generated from rpc ctrl.v1.ClusterService.Heartbeat + */ + heartbeat: { + methodKind: "unary"; + input: typeof HeartbeatRequestSchema; + output: typeof HeartbeatResponseSchema; + }, }> = /*@__PURE__*/ serviceDesc(file_ctrl_v1_cluster, 0); diff --git a/web/apps/dashboard/lib/collections/deploy/environment-settings.ts b/web/apps/dashboard/lib/collections/deploy/environment-settings.ts index a8a2380e1ff..9bb28312cca 100644 --- a/web/apps/dashboard/lib/collections/deploy/environment-settings.ts +++ b/web/apps/dashboard/lib/collections/deploy/environment-settings.ts @@ -42,7 +42,7 @@ const schema = z.object({ memoryMib: z.number().int(), command: z.array(z.string()), healthcheck: healthcheckSchema, - regionConfig: z.record(z.string(), z.number()), + regions: z.array(z.object({ id: z.string(), name: z.string(), replicas: z.number().int() })), shutdownSignal: z.string(), sentinelConfig: sentinelConfigSchema, }); @@ -80,7 +80,14 @@ export const environmentSettings = createCollection environmentId, }); - return [flattenSettingsResponse(environmentId, result.buildSettings, result.runtimeSettings)]; + return [ + flattenSettingsResponse( + environmentId, + result.buildSettings, + result.runtimeSettings, + result.regionalSettings, + ), + ]; }, getKey: (item) => item.environmentId, id: "environmentSettings", @@ -108,6 +115,7 @@ function flattenSettingsResponse( environmentId: string, build: SettingsResponse["buildSettings"], runtime: SettingsResponse["runtimeSettings"], + regional: SettingsResponse["regionalSettings"], ): EnvironmentSettings { return { environmentId, @@ -118,7 +126,11 @@ function flattenSettingsResponse( memoryMib: runtime?.memoryMib ?? 256, command: runtime?.command ?? [], healthcheck: runtime?.healthcheck ?? null, - regionConfig: runtime?.regionConfig ?? {}, + regions: regional.map((r) => ({ + id: r.region.id, + name: r.region.name, + replicas: r.replicas, + })), shutdownSignal: "SIGTERM", sentinelConfig: runtime?.sentinelConfig, }; @@ -200,32 +212,29 @@ export function buildSettingsMutations( ); } - const origRegions = Object.keys(original.regionConfig).sort(); - const modRegions = Object.keys(modified.regionConfig).sort(); - const regionsChanged = changed(origRegions, modRegions); + const origRegionIds = original.regions.map((r) => r.id).sort(); + const modRegionIds = modified.regions.map((r) => r.id).sort(); + const regionsChanged = changed(origRegionIds, modRegionIds); if (regionsChanged) { mutations.push( trpcClient.deploy.environmentSettings.runtime.updateRegions.mutate({ environmentId, - regions: modRegions, + regionIds: modRegionIds, }), ); } - const origValues = Object.values(original.regionConfig); - const modValues = Object.values(modified.regionConfig); + const origReplicas = original.regions.at(0)?.replicas ?? 1; + const modReplicas = modified.regions.at(0)?.replicas ?? 1; const instancesChanged = - !regionsChanged && - origValues.length === modValues.length && - modValues.length > 0 && - modValues[0] !== origValues[0]; + !regionsChanged && modified.regions.length > 0 && modReplicas !== origReplicas; if (instancesChanged) { mutations.push( trpcClient.deploy.environmentSettings.runtime.updateInstances.mutate({ environmentId, - replicasPerRegion: modValues[0], + replicasPerRegion: modReplicas, }), ); } diff --git a/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/get-available-regions.ts b/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/get-available-regions.ts index bc0999979b7..4dd66742248 100644 --- a/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/get-available-regions.ts +++ b/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/get-available-regions.ts @@ -1,9 +1,9 @@ +import { db } from "@/lib/db"; import { workspaceProcedure } from "../../../trpc"; -export const getAvailableRegions = workspaceProcedure.query(() => { - const regionsEnv = process.env.AVAILABLE_REGIONS ?? ""; - return regionsEnv - .split(",") - .map((r) => r.trim()) - .filter(Boolean); +export const getAvailableRegions = workspaceProcedure.query(async () => { + const r = await db.query.regions.findMany({ + columns: { id: true, name: true }, + }); + return r; }); diff --git a/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/get.ts b/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/get.ts index 56855bdcbb4..bddb3bac1a8 100644 --- a/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/get.ts +++ b/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/get.ts @@ -1,5 +1,5 @@ import { and, db, eq } from "@/lib/db"; -import { appBuildSettings, appRuntimeSettings } from "@unkey/db/src/schema"; +import { appBuildSettings, appRegionalSettings, appRuntimeSettings } from "@unkey/db/src/schema"; import { z } from "zod"; import { workspaceProcedure } from "../../../trpc"; import type { SentinelConfig } from "./sentinel/update-middleware"; @@ -7,7 +7,7 @@ import type { SentinelConfig } from "./sentinel/update-middleware"; export const getEnvironmentSettings = workspaceProcedure .input(z.object({ environmentId: z.string() })) .query(async ({ ctx, input }) => { - const [buildSettings, runtimeSettings] = await Promise.all([ + const [buildSettings, runtimeSettings, regionalSettings] = await Promise.all([ db.query.appBuildSettings.findFirst({ where: and( eq(appBuildSettings.workspaceId, ctx.workspace.id), @@ -20,6 +20,15 @@ export const getEnvironmentSettings = workspaceProcedure eq(appRuntimeSettings.environmentId, input.environmentId), ), }), + db.query.appRegionalSettings.findMany({ + where: and( + eq(appRegionalSettings.workspaceId, ctx.workspace.id), + eq(appRegionalSettings.environmentId, input.environmentId), + ), + with: { + region: true, + }, + }), ]); return { @@ -35,5 +44,6 @@ export const getEnvironmentSettings = workspaceProcedure : undefined, } : null, + regionalSettings, }; }); diff --git a/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/runtime/update-instances.ts b/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/runtime/update-instances.ts index 369cf7842c3..b3f5919a0fd 100644 --- a/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/runtime/update-instances.ts +++ b/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/runtime/update-instances.ts @@ -1,5 +1,5 @@ import { and, db, eq, inArray } from "@/lib/db"; -import { appRuntimeSettings } from "@unkey/db/src/schema"; +import { appRegionalSettings } from "@unkey/db/src/schema"; import { z } from "zod"; import { workspaceProcedure } from "../../../../trpc"; import { resolveProjectEnvironmentIds } from "../utils"; @@ -14,36 +14,15 @@ export const updateInstances = workspaceProcedure .mutation(async ({ ctx, input }) => { const envIds = await resolveProjectEnvironmentIds(ctx.workspace.id, input.environmentId); - const existing = await db.query.appRuntimeSettings.findFirst({ - where: and( - eq(appRuntimeSettings.workspaceId, ctx.workspace.id), - eq(appRuntimeSettings.environmentId, input.environmentId), - ), - }); - - const currentConfig = (existing?.regionConfig as Record) ?? {}; - const currentRegions = Object.keys(currentConfig); - - const regionConfig: Record = {}; - - if (currentRegions.length > 0) { - for (const region of currentRegions) { - regionConfig[region] = input.replicasPerRegion; - } - } else { - const regionsEnv = process.env.AVAILABLE_REGIONS ?? ""; - for (const region of regionsEnv.split(",")) { - regionConfig[region] = input.replicasPerRegion; - } - } - await db - .update(appRuntimeSettings) - .set({ regionConfig }) + .update(appRegionalSettings) + .set({ + replicas: input.replicasPerRegion, + }) .where( and( - eq(appRuntimeSettings.workspaceId, ctx.workspace.id), - inArray(appRuntimeSettings.environmentId, envIds), + eq(appRegionalSettings.workspaceId, ctx.workspace.id), + inArray(appRegionalSettings.environmentId, envIds), ), ); }); diff --git a/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/runtime/update-regions.ts b/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/runtime/update-regions.ts index ef4a557a039..39ef1a0251c 100644 --- a/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/runtime/update-regions.ts +++ b/web/apps/dashboard/lib/trpc/routers/deploy/environment-settings/runtime/update-regions.ts @@ -1,5 +1,6 @@ -import { and, db, eq, inArray } from "@/lib/db"; -import { appRuntimeSettings } from "@unkey/db/src/schema"; +import { and, db, eq, notInArray } from "@/lib/db"; +import { TRPCError } from "@trpc/server"; +import { appRegionalSettings, environments } from "@unkey/db/src/schema"; import { z } from "zod"; import { workspaceProcedure } from "../../../../trpc"; import { resolveProjectEnvironmentIds } from "../utils"; @@ -8,32 +9,60 @@ export const updateRegions = workspaceProcedure .input( z.object({ environmentId: z.string(), - regions: z.array(z.string()).min(1), + regionIds: z.array(z.string()).min(1), }), ) .mutation(async ({ ctx, input }) => { const envIds = await resolveProjectEnvironmentIds(ctx.workspace.id, input.environmentId); - const existing = await db.query.appRuntimeSettings.findFirst({ + // Get appId from the environment + const env = await db.query.environments.findFirst({ where: and( - eq(appRuntimeSettings.workspaceId, ctx.workspace.id), - eq(appRuntimeSettings.environmentId, input.environmentId), + eq(environments.id, input.environmentId), + eq(environments.workspaceId, ctx.workspace.id), ), + columns: { appId: true }, }); - - const currentConfig = (existing?.regionConfig as Record) ?? {}; - const regionConfig: Record = {}; - for (const region of input.regions) { - regionConfig[region] = currentConfig[region] ?? 1; + if (!env) { + throw new TRPCError({ code: "NOT_FOUND", message: "Environment not found" }); } - await db - .update(appRuntimeSettings) - .set({ regionConfig }) - .where( - and( - eq(appRuntimeSettings.workspaceId, ctx.workspace.id), - inArray(appRuntimeSettings.environmentId, envIds), - ), - ); + // Get existing regional settings to preserve replica counts for existing regions + const existingSettings = await db.query.appRegionalSettings.findMany({ + where: and( + eq(appRegionalSettings.workspaceId, ctx.workspace.id), + eq(appRegionalSettings.environmentId, input.environmentId), + ), + }); + + const defaults = existingSettings.at(0) ?? { + replicas: 1, + }; + + for (const envId of envIds) { + // Delete rows for regions that are no longer selected + await db + .delete(appRegionalSettings) + .where( + and( + eq(appRegionalSettings.workspaceId, ctx.workspace.id), + eq(appRegionalSettings.environmentId, envId), + notInArray(appRegionalSettings.regionId, input.regionIds), + ), + ); + + // Insert rows for newly added regions + for (const regionId of input.regionIds) { + const existing = existingSettings.find((s) => s.regionId === regionId); + if (!existing) { + await db.insert(appRegionalSettings).values({ + workspaceId: ctx.workspace.id, + appId: env.appId, + environmentId: envId, + regionId, + replicas: defaults.replicas, + }); + } + } + } }); diff --git a/web/apps/dashboard/lib/trpc/routers/deploy/network/utils.ts b/web/apps/dashboard/lib/trpc/routers/deploy/network/utils.ts index 2e1a8f16a9e..574d5d9a21d 100644 --- a/web/apps/dashboard/lib/trpc/routers/deploy/network/utils.ts +++ b/web/apps/dashboard/lib/trpc/routers/deploy/network/utils.ts @@ -1,7 +1,7 @@ import type { HealthStatus } from "@/app/(app)/[workspaceSlug]/projects/[projectId]/(overview)/deployments/[deploymentId]/network/unkey-flow/components/nodes/types"; import type { Instance, Sentinel } from "@/lib/db"; -export const flagCodes = ["us", "de", "au", "jp", "in", "br"] as const; +export const flagCodes = ["us", "de", "au", "jp", "in", "br", "local"] as const; export type FlagCode = (typeof flagCodes)[number]; export function mapInstanceStatusToHealth(status: Instance["status"]): HealthStatus { @@ -50,6 +50,7 @@ export function mapRegionToFlag(region: string): FlagCode { ["us-", "us"], ["eu-", "de"], ["sa-", "br"], + ["local", "local"], ] as const; return (prefixMap.find(([prefix]) => region.startsWith(prefix))?.[1] as FlagCode) ?? "us"; diff --git a/web/internal/db/src/schema/app_regional_settings.ts b/web/internal/db/src/schema/app_regional_settings.ts new file mode 100644 index 00000000000..5d89ab83e9b --- /dev/null +++ b/web/internal/db/src/schema/app_regional_settings.ts @@ -0,0 +1,57 @@ +import { relations } from "drizzle-orm"; +import { bigint, index, int, mysqlTable, uniqueIndex, varchar } from "drizzle-orm/mysql-core"; +import { apps } from "./apps"; +import { environments } from "./environments"; +import { regions } from "./regions"; +import { lifecycleDates } from "./util/lifecycle_dates"; +import { workspaces } from "./workspaces"; + +/** + * One row per region an app+environment should be deployed to. + * Replaces the `region_config` JSON column on `app_runtime_settings`. + * + * Presence of a row means "deploy this app/env to this region". + */ +export const appRegionalSettings = mysqlTable( + "app_regional_settings", + { + pk: bigint("pk", { mode: "number", unsigned: true }).autoincrement().primaryKey(), + + workspaceId: varchar("workspace_id", { length: 256 }).notNull(), + appId: varchar("app_id", { length: 64 }).notNull(), + environmentId: varchar("environment_id", { length: 128 }).notNull(), + regionId: varchar("region_id", { length: 64 }).notNull(), + + replicas: int("replicas").notNull().default(1), + + // Optional reference to a horizontal autoscaling policy. null = no autoscaling. + horizontalAutoscalingPolicyId: varchar("horizontal_autoscaling_policy_id", { + length: 64, + }), + + ...lifecycleDates, + }, + (table) => [ + uniqueIndex("unique_app_env_region").on(table.appId, table.environmentId, table.regionId), + index("workspace_idx").on(table.workspaceId), + ], +); + +export const appRegionalSettingsRelations = relations(appRegionalSettings, ({ one }) => ({ + workspace: one(workspaces, { + fields: [appRegionalSettings.workspaceId], + references: [workspaces.id], + }), + app: one(apps, { + fields: [appRegionalSettings.appId], + references: [apps.id], + }), + environment: one(environments, { + fields: [appRegionalSettings.environmentId], + references: [environments.id], + }), + region: one(regions, { + fields: [appRegionalSettings.regionId], + references: [regions.id], + }), +})); diff --git a/web/internal/db/src/schema/app_runtime_settings.ts b/web/internal/db/src/schema/app_runtime_settings.ts index 3492e902e30..aa7b0ddb4d7 100644 --- a/web/internal/db/src/schema/app_runtime_settings.ts +++ b/web/internal/db/src/schema/app_runtime_settings.ts @@ -42,6 +42,7 @@ export const appRuntimeSettings = mysqlTable( // Maps region ID to replica count, e.g. {"us-east-1": 3, "eu-central-1": 1} // Empty object = 1 replica in all available regions (default behavior) + // @deprecated, use appRegionalSettings instead regionConfig: json("region_config") .$type>() .notNull() diff --git a/web/internal/db/src/schema/clusters.ts b/web/internal/db/src/schema/clusters.ts new file mode 100644 index 00000000000..d60812e4bfa --- /dev/null +++ b/web/internal/db/src/schema/clusters.ts @@ -0,0 +1,26 @@ +import { relations } from "drizzle-orm"; +import { bigint, mysqlTable, varchar } from "drizzle-orm/mysql-core"; +import { regions } from "./regions"; + +// clusters tracks our kubernetes clusters +// each krane instance will heartbeat against the control plane, which then writes to this table +// +// we might use this as service discovery later to push updates to clusters to speed up reconciliation +export const clusters = mysqlTable("clusters", { + pk: bigint("pk", { mode: "number", unsigned: true }).autoincrement().primaryKey(), + + id: varchar("id", { length: 64 }).notNull().unique(), + regionId: varchar("region_id", { length: 64 }).notNull().unique(), + + lastHeartbeatAt: bigint("last_heartbeat_at", { + mode: "number", + unsigned: true, + }).notNull(), +}); + +export const clustersRelations = relations(clusters, ({ one }) => ({ + region: one(regions, { + fields: [clusters.regionId], + references: [regions.id], + }), +})); diff --git a/web/internal/db/src/schema/deployment_topology.ts b/web/internal/db/src/schema/deployment_topology.ts index 37ab0382d56..445dfe6debf 100644 --- a/web/internal/db/src/schema/deployment_topology.ts +++ b/web/internal/db/src/schema/deployment_topology.ts @@ -20,13 +20,14 @@ export const deploymentTopology = mysqlTable( deploymentId: varchar("deployment_id", { length: 64 }).notNull(), region: varchar("region", { length: 64 }).notNull(), + regionId: varchar("region_id", { length: 64 }).notNull().default(""), desiredReplicas: int("desired_replicas").notNull(), // Version for state synchronization with edge agents. // Updated via Restate VersioningService on each mutation. // Edge agents track their last-seen version and request changes after it. - // Unique per region (composite index with region). + // Unique per regionId (composite index with regionId). version: bigint("version", { mode: "number", unsigned: true }).notNull(), // Deployment status @@ -35,12 +36,13 @@ export const deploymentTopology = mysqlTable( "started", "stopping", "stopped", + "running", ]).notNull(), ...lifecycleDates, }, (table) => [ - uniqueIndex("unique_region_per_deployment").on(table.deploymentId, table.region), - uniqueIndex("unique_version_per_region").on(table.region, table.version), + uniqueIndex("unique_region_per_deployment").on(table.deploymentId, table.regionId), + uniqueIndex("unique_version_per_region").on(table.regionId, table.version), index("workspace_idx").on(table.workspaceId), index("status_idx").on(table.desiredStatus), ], diff --git a/web/internal/db/src/schema/horizontal_autoscaling_policies.ts b/web/internal/db/src/schema/horizontal_autoscaling_policies.ts new file mode 100644 index 00000000000..fdf413091ff --- /dev/null +++ b/web/internal/db/src/schema/horizontal_autoscaling_policies.ts @@ -0,0 +1,41 @@ +import { relations } from "drizzle-orm"; +import { bigint, index, int, mysqlTable, tinyint, varchar } from "drizzle-orm/mysql-core"; +import { lifecycleDates } from "./util/lifecycle_dates"; +import { workspaces } from "./workspaces"; + +/** + * A reusable horizontal autoscaling policy. + * + * Other tables (e.g. app_regional_settings, sentinels) reference this via horizontalAutoscalingPolicyId. + * If no policy is referenced, static replica counts are used. + */ +export const horizontalAutoscalingPolicies = mysqlTable( + "horizontal_autoscaling_policies", + { + pk: bigint("pk", { mode: "number", unsigned: true }).autoincrement().primaryKey(), + id: varchar("id", { length: 64 }).notNull().unique(), + + workspaceId: varchar("workspace_id", { length: 256 }).notNull(), + + replicasMin: int("replicas_min").notNull(), + replicasMax: int("replicas_max").notNull(), + + // 0-100, percentage thresholds that trigger scaling. null = not used as a signal. + memoryThreshold: tinyint("memory_threshold"), + cpuThreshold: tinyint("cpu_threshold"), + rpsThreshold: tinyint("rps_threshold"), + + ...lifecycleDates, + }, + (table) => [index("workspace_idx").on(table.workspaceId)], +); + +export const horizontalAutoscalingPoliciesRelations = relations( + horizontalAutoscalingPolicies, + ({ one }) => ({ + workspace: one(workspaces, { + fields: [horizontalAutoscalingPolicies.workspaceId], + references: [workspaces.id], + }), + }), +); diff --git a/web/internal/db/src/schema/index.ts b/web/internal/db/src/schema/index.ts index 415bd9b0579..cfa33e5041d 100644 --- a/web/internal/db/src/schema/index.ts +++ b/web/internal/db/src/schema/index.ts @@ -17,6 +17,8 @@ export * from "./projects"; export * from "./apps"; export * from "./app_build_settings"; export * from "./app_runtime_settings"; +export * from "./app_regional_settings"; + export * from "./app_environment_variables"; export * from "./deployments"; export * from "./deployment_steps"; @@ -32,3 +34,7 @@ export * from "./certificates"; export * from "./frontline_routes"; export * from "./github_app"; export * from "./cilium"; + +export * from "./clusters"; +export * from "./regions"; +export * from "./horizontal_autoscaling_policies"; diff --git a/web/internal/db/src/schema/regions.ts b/web/internal/db/src/schema/regions.ts new file mode 100644 index 00000000000..d7bf1e76807 --- /dev/null +++ b/web/internal/db/src/schema/regions.ts @@ -0,0 +1,20 @@ +import { relations } from "drizzle-orm"; +import { bigint, mysqlTable, uniqueIndex, varchar } from "drizzle-orm/mysql-core"; +import { clusters } from "./clusters"; + +export const regions = mysqlTable( + "regions", + { + pk: bigint("pk", { mode: "number", unsigned: true }).autoincrement().primaryKey(), + id: varchar("id", { length: 64 }).notNull().unique(), + // e.g. us-east-1, us-west-2, etc. + name: varchar("name", { length: 64 }).notNull(), + // e.g. aws, gcp, azure, local, etc. + platform: varchar("platform", { length: 64 }).notNull(), + }, + (table) => [uniqueIndex("unique_region_per_platform").on(table.name, table.platform)], +); + +export const regionRelations = relations(regions, ({ many }) => ({ + clusters: many(clusters), +})); diff --git a/web/tools/migrate/region-config-to-regional-settings.ts b/web/tools/migrate/region-config-to-regional-settings.ts new file mode 100644 index 00000000000..9413ce46711 --- /dev/null +++ b/web/tools/migrate/region-config-to-regional-settings.ts @@ -0,0 +1,76 @@ +import { mysqlDrizzle, schema } from "@unkey/db"; +import mysql from "mysql2/promise"; + +/** + * Migrates region_config JSON from app_runtime_settings into + * individual app_regional_settings rows. + * + * For each app_runtime_settings row with a non-empty region_config, + * creates one app_regional_settings row per region entry. + * + * Safe to run multiple times — uses INSERT IGNORE to skip existing rows. + */ +async function main() { + const conn = await mysql.createConnection( + `mysql://${process.env.DATABASE_USERNAME}:${process.env.DATABASE_PASSWORD}@${process.env.DATABASE_HOST}:3306/unkey?ssl={}`, + ); + + await conn.ping(); + const db = mysqlDrizzle(conn, { schema, mode: "default" }); + + const runtimeSettings = await db.query.appRuntimeSettings.findMany({ + columns: { + workspaceId: true, + appId: true, + environmentId: true, + regionConfig: true, + }, + }); + + console.info(`Found ${runtimeSettings.length} runtime settings rows`); + + let created = 0; + let skipped = 0; + + for (const row of runtimeSettings) { + const regionConfig = (row.regionConfig ?? {}) as Record; + const regions = Object.entries(regionConfig); + + if (regions.length === 0) { + skipped++; + continue; + } + + for (const [regionId, replicas] of regions) { + try { + await db.insert(schema.appRegionalSettings).values({ + workspaceId: row.workspaceId, + appId: row.appId, + environmentId: row.environmentId, + regionId, + replicas, + createdAt: Date.now(), + updatedAt: Date.now(), + }); + created++; + console.info( + `Created: app=${row.appId} env=${row.environmentId} region=${regionId} replicas=${replicas}`, + ); + } catch (err: unknown) { + if (err instanceof Error && err.message.includes("Duplicate entry")) { + console.info( + `Skipped (already exists): app=${row.appId} env=${row.environmentId} region=${regionId}`, + ); + skipped++; + } else { + throw err; + } + } + } + } + + console.info(`Migration complete. Created: ${created}, Skipped: ${skipped}`); + await conn.end(); +} + +main();