diff --git a/go/pkg/db/bulk_deployment_insert.sql_generated.go b/go/pkg/db/bulk_deployment_insert.sql_generated.go index 771e97d7e5..ae5410b3b4 100644 --- a/go/pkg/db/bulk_deployment_insert.sql_generated.go +++ b/go/pkg/db/bulk_deployment_insert.sql_generated.go @@ -9,7 +9,7 @@ import ( ) // bulkInsertDeployment is the base query for bulk insert -const bulkInsertDeployment = `INSERT INTO ` + "`" + `deployments` + "`" + ` ( id, workspace_id, project_id, environment_id, git_commit_sha, git_branch, runtime_config, gateway_config, git_commit_message, git_commit_author_handle, git_commit_author_avatar_url, git_commit_timestamp, openapi_spec, status, gateway_config, created_at, updated_at ) VALUES %s` +const bulkInsertDeployment = `INSERT INTO ` + "`" + `deployments` + "`" + ` ( id, workspace_id, project_id, environment_id, git_commit_sha, git_branch, runtime_config, gateway_config, git_commit_message, git_commit_author_handle, git_commit_author_avatar_url, git_commit_timestamp, openapi_spec, secrets_config, status, created_at, updated_at ) VALUES %s` // InsertDeployments performs bulk insert in a single query func (q *BulkQueries) InsertDeployments(ctx context.Context, db DBTX, args []InsertDeploymentParams) error { @@ -42,8 +42,8 @@ func (q *BulkQueries) InsertDeployments(ctx context.Context, db DBTX, args []Ins allArgs = append(allArgs, arg.GitCommitAuthorAvatarUrl) allArgs = append(allArgs, arg.GitCommitTimestamp) allArgs = append(allArgs, arg.OpenapiSpec) + allArgs = append(allArgs, arg.SecretsConfig) allArgs = append(allArgs, arg.Status) - allArgs = append(allArgs, arg.GatewayConfig) allArgs = append(allArgs, arg.CreatedAt) allArgs = append(allArgs, arg.UpdatedAt) } diff --git a/go/pkg/db/deployment_find_by_id.sql_generated.go b/go/pkg/db/deployment_find_by_id.sql_generated.go index a413f36715..d3156b74bf 100644 --- a/go/pkg/db/deployment_find_by_id.sql_generated.go +++ b/go/pkg/db/deployment_find_by_id.sql_generated.go @@ -10,12 +10,12 @@ import ( ) const findDeploymentById = `-- name: FindDeploymentById :one -SELECT id, workspace_id, project_id, environment_id, git_commit_sha, git_branch, git_commit_message, git_commit_author_handle, git_commit_author_avatar_url, git_commit_timestamp, runtime_config, gateway_config, openapi_spec, status, created_at, updated_at FROM ` + "`" + `deployments` + "`" + ` WHERE id = ? +SELECT id, workspace_id, project_id, environment_id, git_commit_sha, git_branch, git_commit_message, git_commit_author_handle, git_commit_author_avatar_url, git_commit_timestamp, runtime_config, gateway_config, openapi_spec, secrets_config, status, created_at, updated_at FROM ` + "`" + `deployments` + "`" + ` WHERE id = ? ` // FindDeploymentById // -// SELECT id, workspace_id, project_id, environment_id, git_commit_sha, git_branch, git_commit_message, git_commit_author_handle, git_commit_author_avatar_url, git_commit_timestamp, runtime_config, gateway_config, openapi_spec, status, created_at, updated_at FROM `deployments` WHERE id = ? +// SELECT id, workspace_id, project_id, environment_id, git_commit_sha, git_branch, git_commit_message, git_commit_author_handle, git_commit_author_avatar_url, git_commit_timestamp, runtime_config, gateway_config, openapi_spec, secrets_config, status, created_at, updated_at FROM `deployments` WHERE id = ? func (q *Queries) FindDeploymentById(ctx context.Context, db DBTX, id string) (Deployment, error) { row := db.QueryRowContext(ctx, findDeploymentById, id) var i Deployment @@ -33,6 +33,7 @@ func (q *Queries) FindDeploymentById(ctx context.Context, db DBTX, id string) (D &i.RuntimeConfig, &i.GatewayConfig, &i.OpenapiSpec, + &i.SecretsConfig, &i.Status, &i.CreatedAt, &i.UpdatedAt, diff --git a/go/pkg/db/deployment_insert.sql_generated.go b/go/pkg/db/deployment_insert.sql_generated.go index 95e368905a..2d7237d6d3 100644 --- a/go/pkg/db/deployment_insert.sql_generated.go +++ b/go/pkg/db/deployment_insert.sql_generated.go @@ -24,10 +24,10 @@ INSERT INTO ` + "`" + `deployments` + "`" + ` ( git_commit_message, git_commit_author_handle, git_commit_author_avatar_url, - git_commit_timestamp, -- Unix epoch milliseconds + git_commit_timestamp, openapi_spec, + secrets_config, status, - gateway_config, created_at, updated_at ) @@ -66,6 +66,7 @@ type InsertDeploymentParams struct { GitCommitAuthorAvatarUrl sql.NullString `db:"git_commit_author_avatar_url"` GitCommitTimestamp sql.NullInt64 `db:"git_commit_timestamp"` OpenapiSpec sql.NullString `db:"openapi_spec"` + SecretsConfig []byte `db:"secrets_config"` Status DeploymentsStatus `db:"status"` CreatedAt int64 `db:"created_at"` UpdatedAt sql.NullInt64 `db:"updated_at"` @@ -85,10 +86,10 @@ type InsertDeploymentParams struct { // git_commit_message, // git_commit_author_handle, // git_commit_author_avatar_url, -// git_commit_timestamp, -- Unix epoch milliseconds +// git_commit_timestamp, // openapi_spec, +// secrets_config, // status, -// gateway_config, // created_at, // updated_at // ) @@ -126,8 +127,8 @@ func (q *Queries) InsertDeployment(ctx context.Context, db DBTX, arg InsertDeplo arg.GitCommitAuthorAvatarUrl, arg.GitCommitTimestamp, arg.OpenapiSpec, + arg.SecretsConfig, arg.Status, - arg.GatewayConfig, arg.CreatedAt, arg.UpdatedAt, ) diff --git a/go/pkg/db/environment_variables_find_by_environment_id.sql_generated.go b/go/pkg/db/environment_variables_find_by_environment_id.sql_generated.go new file mode 100644 index 0000000000..590d40c8f4 --- /dev/null +++ b/go/pkg/db/environment_variables_find_by_environment_id.sql_generated.go @@ -0,0 +1,49 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.29.0 +// source: environment_variables_find_by_environment_id.sql + +package db + +import ( + "context" +) + +const findEnvironmentVariablesByEnvironmentId = `-- name: FindEnvironmentVariablesByEnvironmentId :many +SELECT ` + "`" + `key` + "`" + `, value +FROM environment_variables +WHERE environment_id = ? +` + +type FindEnvironmentVariablesByEnvironmentIdRow struct { + Key string `db:"key"` + Value string `db:"value"` +} + +// FindEnvironmentVariablesByEnvironmentId +// +// SELECT `key`, value +// FROM environment_variables +// WHERE environment_id = ? +func (q *Queries) FindEnvironmentVariablesByEnvironmentId(ctx context.Context, db DBTX, environmentID string) ([]FindEnvironmentVariablesByEnvironmentIdRow, error) { + rows, err := db.QueryContext(ctx, findEnvironmentVariablesByEnvironmentId, environmentID) + if err != nil { + return nil, err + } + defer rows.Close() + var items []FindEnvironmentVariablesByEnvironmentIdRow + for rows.Next() { + var i FindEnvironmentVariablesByEnvironmentIdRow + if err := rows.Scan(&i.Key, &i.Value); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} diff --git a/go/pkg/db/models_generated.go b/go/pkg/db/models_generated.go index 231a63474b..5b3416d198 100644 --- a/go/pkg/db/models_generated.go +++ b/go/pkg/db/models_generated.go @@ -278,6 +278,48 @@ func (ns NullDeploymentsStatus) Value() (driver.Value, error) { return string(ns.DeploymentsStatus), nil } +type EnvironmentVariablesType string + +const ( + EnvironmentVariablesTypeRecoverable EnvironmentVariablesType = "recoverable" + EnvironmentVariablesTypeWriteonly EnvironmentVariablesType = "writeonly" +) + +func (e *EnvironmentVariablesType) Scan(src interface{}) error { + switch s := src.(type) { + case []byte: + *e = EnvironmentVariablesType(s) + case string: + *e = EnvironmentVariablesType(s) + default: + return fmt.Errorf("unsupported scan type for EnvironmentVariablesType: %T", src) + } + return nil +} + +type NullEnvironmentVariablesType struct { + EnvironmentVariablesType EnvironmentVariablesType + Valid bool // Valid is true if EnvironmentVariablesType is not NULL +} + +// Scan implements the Scanner interface. +func (ns *NullEnvironmentVariablesType) Scan(value interface{}) error { + if value == nil { + ns.EnvironmentVariablesType, ns.Valid = "", false + return nil + } + ns.Valid = true + return ns.EnvironmentVariablesType.Scan(value) +} + +// Value implements the driver Valuer interface. +func (ns NullEnvironmentVariablesType) Value() (driver.Value, error) { + if !ns.Valid { + return nil, nil + } + return string(ns.EnvironmentVariablesType), nil +} + type GatewaysHealth string const ( @@ -747,6 +789,7 @@ type Deployment struct { RuntimeConfig json.RawMessage `db:"runtime_config"` GatewayConfig []byte `db:"gateway_config"` OpenapiSpec sql.NullString `db:"openapi_spec"` + SecretsConfig []byte `db:"secrets_config"` Status DeploymentsStatus `db:"status"` CreatedAt int64 `db:"created_at"` UpdatedAt sql.NullInt64 `db:"updated_at"` @@ -782,6 +825,19 @@ type Environment struct { UpdatedAt sql.NullInt64 `db:"updated_at"` } +type EnvironmentVariable struct { + ID string `db:"id"` + WorkspaceID string `db:"workspace_id"` + EnvironmentID string `db:"environment_id"` + Key string `db:"key"` + Value string `db:"value"` + Type EnvironmentVariablesType `db:"type"` + Description sql.NullString `db:"description"` + DeleteProtection sql.NullBool `db:"delete_protection"` + CreatedAt int64 `db:"created_at"` + UpdatedAt sql.NullInt64 `db:"updated_at"` +} + type Gateway struct { ID string `db:"id"` WorkspaceID string `db:"workspace_id"` diff --git a/go/pkg/db/querier_generated.go b/go/pkg/db/querier_generated.go index e0fe39c1eb..e0cbf969cd 100644 --- a/go/pkg/db/querier_generated.go +++ b/go/pkg/db/querier_generated.go @@ -190,7 +190,7 @@ type Querier interface { FindCustomDomainById(ctx context.Context, db DBTX, id string) (FindCustomDomainByIdRow, error) //FindDeploymentById // - // SELECT id, workspace_id, project_id, environment_id, git_commit_sha, git_branch, git_commit_message, git_commit_author_handle, git_commit_author_avatar_url, git_commit_timestamp, runtime_config, gateway_config, openapi_spec, status, created_at, updated_at FROM `deployments` WHERE id = ? + // SELECT id, workspace_id, project_id, environment_id, git_commit_sha, git_branch, git_commit_message, git_commit_author_handle, git_commit_author_avatar_url, git_commit_timestamp, runtime_config, gateway_config, openapi_spec, secrets_config, status, created_at, updated_at FROM `deployments` WHERE id = ? FindDeploymentById(ctx context.Context, db DBTX, id string) (Deployment, error) //FindDeploymentStepsByDeploymentId // @@ -217,6 +217,12 @@ type Querier interface { // AND project_id = ? // AND slug = ? FindEnvironmentByProjectIdAndSlug(ctx context.Context, db DBTX, arg FindEnvironmentByProjectIdAndSlugParams) (Environment, error) + //FindEnvironmentVariablesByEnvironmentId + // + // SELECT `key`, value + // FROM environment_variables + // WHERE environment_id = ? + FindEnvironmentVariablesByEnvironmentId(ctx context.Context, db DBTX, environmentID string) ([]FindEnvironmentVariablesByEnvironmentIdRow, error) //FindGatewaysByEnvironmentID // // SELECT id, workspace_id, environment_id, k8s_service_name, region, image, health, replicas FROM gateways WHERE environment_id = ? @@ -1108,10 +1114,10 @@ type Querier interface { // git_commit_message, // git_commit_author_handle, // git_commit_author_avatar_url, - // git_commit_timestamp, -- Unix epoch milliseconds + // git_commit_timestamp, // openapi_spec, + // secrets_config, // status, - // gateway_config, // created_at, // updated_at // ) diff --git a/go/pkg/db/queries/deployment_insert.sql b/go/pkg/db/queries/deployment_insert.sql index 4473559101..2bf552a811 100644 --- a/go/pkg/db/queries/deployment_insert.sql +++ b/go/pkg/db/queries/deployment_insert.sql @@ -11,10 +11,10 @@ INSERT INTO `deployments` ( git_commit_message, git_commit_author_handle, git_commit_author_avatar_url, - git_commit_timestamp, -- Unix epoch milliseconds + git_commit_timestamp, openapi_spec, + secrets_config, status, - gateway_config, created_at, updated_at ) @@ -32,8 +32,8 @@ VALUES ( sqlc.arg(git_commit_author_avatar_url), sqlc.arg(git_commit_timestamp), sqlc.arg(openapi_spec), + sqlc.arg(secrets_config), sqlc.arg(status), - sqlc.arg(gateway_config), sqlc.arg(created_at), sqlc.arg(updated_at) ); diff --git a/go/pkg/db/queries/environment_variables_find_by_environment_id.sql b/go/pkg/db/queries/environment_variables_find_by_environment_id.sql new file mode 100644 index 0000000000..866a56b25f --- /dev/null +++ b/go/pkg/db/queries/environment_variables_find_by_environment_id.sql @@ -0,0 +1,4 @@ +-- name: FindEnvironmentVariablesByEnvironmentId :many +SELECT `key`, value +FROM environment_variables +WHERE environment_id = sqlc.arg(environment_id); diff --git a/go/pkg/db/schema.sql b/go/pkg/db/schema.sql index d856d164bd..66eccc8da9 100644 --- a/go/pkg/db/schema.sql +++ b/go/pkg/db/schema.sql @@ -316,6 +316,21 @@ CREATE TABLE `environments` ( CONSTRAINT `environments_project_id_slug_idx` UNIQUE(`project_id`,`slug`) ); +CREATE TABLE `environment_variables` ( + `id` varchar(128) NOT NULL, + `workspace_id` varchar(256) NOT NULL, + `environment_id` varchar(128) NOT NULL, + `key` varchar(256) NOT NULL, + `value` varchar(4096) NOT NULL, + `type` enum('recoverable','writeonly') NOT NULL, + `description` varchar(255), + `delete_protection` boolean DEFAULT false, + `created_at` bigint NOT NULL, + `updated_at` bigint, + CONSTRAINT `environment_variables_id` PRIMARY KEY(`id`), + CONSTRAINT `environment_id_key` UNIQUE(`environment_id`,`key`) +); + CREATE TABLE `clickhouse_workspace_settings` ( `workspace_id` varchar(256) NOT NULL, `username` varchar(256) NOT NULL, @@ -363,6 +378,7 @@ CREATE TABLE `deployments` ( `runtime_config` json NOT NULL, `gateway_config` longblob NOT NULL, `openapi_spec` longblob, + `secrets_config` longblob NOT NULL, `status` enum('pending','building','deploying','network','ready','failed') NOT NULL DEFAULT 'pending', `created_at` bigint NOT NULL, `updated_at` bigint, diff --git a/internal/db/src/schema/deployments.ts b/internal/db/src/schema/deployments.ts index 17bdd0cb07..28e3fbc502 100644 --- a/internal/db/src/schema/deployments.ts +++ b/internal/db/src/schema/deployments.ts @@ -46,6 +46,10 @@ export const deployments = mysqlTable( // OpenAPI specification openapiSpec: longblob("openapi_spec"), + // Environment variables snapshot (protobuf: ctrl.v1.SecretsBlob) + // Encrypted values from environment_variables at deploy time + secretsConfig: longblob("secrets_config").notNull(), + // Deployment status status: mysqlEnum("status", ["pending", "building", "deploying", "network", "ready", "failed"]) .notNull() diff --git a/internal/db/src/schema/environment_variables.ts b/internal/db/src/schema/environment_variables.ts index a5f54b777a..cd2c8cbb33 100644 --- a/internal/db/src/schema/environment_variables.ts +++ b/internal/db/src/schema/environment_variables.ts @@ -5,6 +5,7 @@ import { lifecycleDates } from "./util/lifecycle_dates"; import { workspaces } from "./workspaces"; import { environments } from "./environments"; + export const environmentVariables = mysqlTable( "environment_variables", { @@ -15,9 +16,14 @@ export const environmentVariables = mysqlTable( }).notNull(), key: varchar("key", { length: 256 }).notNull(), - // Either the plaintext value or a vault encrypted response - value: varchar("value", { length: 1024 }).notNull(), - type: mysqlEnum("type", ["plaintext", "secret"]).notNull(), + + // Always encrypted via vault (contains keyId, nonce, ciphertext in the blob) + value: varchar("value", { length: 4096 }).notNull(), + + // Both types are encrypted in the database + // - recoverable: can be decrypted and shown in the UI + // - writeonly: cannot be read back after creation + type: mysqlEnum("type", ["recoverable", "writeonly"]).notNull(), description: varchar("description", { length: 255 }), diff --git a/internal/db/src/schema/index.ts b/internal/db/src/schema/index.ts index 8748aa506d..af0bd61103 100644 --- a/internal/db/src/schema/index.ts +++ b/internal/db/src/schema/index.ts @@ -10,6 +10,7 @@ export * from "./identity"; export * from "./quota"; export * from "./audit_logs"; export * from "./environments"; +export * from "./environment_variables"; export * from "./clickhouse_workspace_settings"; // Deployment platform tables