diff --git a/.github/workflows/job_bazel.yaml b/.github/workflows/job_bazel.yaml index f04af1972a..cf8df657da 100644 --- a/.github/workflows/job_bazel.yaml +++ b/.github/workflows/job_bazel.yaml @@ -33,6 +33,6 @@ jobs: # Running containers is temporary until we moved them inside of bazel, # at that point they are only created if they are actually needed - name: Start containers - run: docker compose -f ./dev/docker-compose.yaml up s3 clickhouse kafka mysql -d --wait + run: docker compose -f ./dev/docker-compose.yaml up s3 clickhouse kafka mysql vault -d --wait - name: Run tests run: bazel test //... --test_output=errors diff --git a/Makefile b/Makefile index 2d733a9a38..65db120cc8 100644 --- a/Makefile +++ b/Makefile @@ -91,7 +91,7 @@ generate: generate-sql ## Generate code from protobuf and other sources .PHONY: test test: ## Run tests with bazel - docker compose -f ./dev/docker-compose.yaml up -d mysql clickhouse s3 kafka --wait + docker compose -f ./dev/docker-compose.yaml up -d mysql clickhouse s3 kafka vault --wait bazel test //... make clean-docker-test diff --git a/cmd/api/main.go b/cmd/api/main.go index f238b9e249..1c03d22dda 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -68,16 +68,10 @@ var Cmd = &cli.Command{ cli.EnvVar("UNKEY_TLS_KEY_FILE")), // Vault Configuration - cli.StringSlice("vault-master-keys", "Vault master keys for encryption", - cli.EnvVar("UNKEY_VAULT_MASTER_KEYS")), - cli.String("vault-s3-url", "S3 Compatible Endpoint URL", - cli.EnvVar("UNKEY_VAULT_S3_URL")), - cli.String("vault-s3-bucket", "S3 bucket name", - cli.EnvVar("UNKEY_VAULT_S3_BUCKET")), - cli.String("vault-s3-access-key-id", "S3 access key ID", - cli.EnvVar("UNKEY_VAULT_S3_ACCESS_KEY_ID")), - cli.String("vault-s3-access-key-secret", "S3 secret access key", - cli.EnvVar("UNKEY_VAULT_S3_ACCESS_KEY_SECRET")), + cli.String("vault-url", "URL of the remote vault service for encryption/decryption", + cli.EnvVar("UNKEY_VAULT_URL")), + cli.String("vault-token", "Bearer token for vault service authentication", + cli.EnvVar("UNKEY_VAULT_TOKEN")), // Kafka Configuration cli.StringSlice("kafka-brokers", "Comma-separated list of Kafka broker addresses for distributed cache invalidation", @@ -146,16 +140,6 @@ func action(ctx context.Context, cmd *cli.Command) error { } } - var vaultS3Config *api.S3Config - if cmd.String("vault-s3-url") != "" { - vaultS3Config = &api.S3Config{ - URL: cmd.String("vault-s3-url"), - Bucket: cmd.String("vault-s3-bucket"), - AccessKeyID: cmd.String("vault-s3-access-key-id"), - AccessKeySecret: cmd.String("vault-s3-access-key-secret"), - } - } - config := api.Config{ // Basic configuration CacheInvalidationTopic: "", @@ -189,8 +173,8 @@ func action(ctx context.Context, cmd *cli.Command) error { Listener: nil, // Production uses HttpPort // Vault configuration - VaultMasterKeys: cmd.StringSlice("vault-master-keys"), - VaultS3: vaultS3Config, + VaultURL: cmd.String("vault-url"), + VaultToken: cmd.String("vault-token"), // Kafka configuration KafkaBrokers: cmd.StringSlice("kafka-brokers"), diff --git a/dev/docker-compose.yaml b/dev/docker-compose.yaml index 40e6a1860a..92ab4b1836 100644 --- a/dev/docker-compose.yaml +++ b/dev/docker-compose.yaml @@ -70,7 +70,7 @@ services: depends_on: mysql: condition: service_healthy - s3: + vault: condition: service_healthy redis: condition: service_healthy @@ -87,11 +87,8 @@ services: UNKEY_CLICKHOUSE_URL: "clickhouse://default:password@clickhouse:9000?secure=false&skip_verify=true" UNKEY_CHPROXY_AUTH_TOKEN: "chproxy-test-token-123" UNKEY_OTEL: false - UNKEY_VAULT_S3_URL: "http://s3:3902" - UNKEY_VAULT_S3_BUCKET: "vault" - UNKEY_VAULT_S3_ACCESS_KEY_ID: "minio_root_user" - UNKEY_VAULT_S3_ACCESS_KEY_SECRET: "minio_root_password" - UNKEY_VAULT_MASTER_KEYS: "Ch9rZWtfMmdqMFBJdVhac1NSa0ZhNE5mOWlLSnBHenFPENTt7an5MRogENt9Si6wms4pQ2XIvqNSIgNpaBenJmXgcInhu6Nfv2U=" + UNKEY_VAULT_URL: "http://vault:8060" + UNKEY_VAULT_TOKEN: "vault-test-token-123" UNKEY_KAFKA_BROKERS: "kafka:9092" UNKEY_CLICKHOUSE_ANALYTICS_URL: "http://clickhouse:8123/default" UNKEY_CTRL_URL: "http://ctrl-api:7091" diff --git a/dev/k8s/manifests/api.yaml b/dev/k8s/manifests/api.yaml index bbfeb9ae10..f179e7989a 100644 --- a/dev/k8s/manifests/api.yaml +++ b/dev/k8s/manifests/api.yaml @@ -56,16 +56,10 @@ spec: - name: UNKEY_PROMETHEUS_PORT value: "0" # Vault Configuration - - name: UNKEY_VAULT_MASTER_KEYS - value: "Ch9rZWtfMmdqMFBJdVhac1NSa0ZhNE5mOWlLSnBHenFPENTt7an5MRogENt9Si6wms4pQ2XIvqNSIgNpaBenJmXgcInhu6Nfv2U=" - - name: UNKEY_VAULT_S3_URL - value: "http://s3:3902" - - name: UNKEY_VAULT_S3_BUCKET - value: "vault" - - name: UNKEY_VAULT_S3_ACCESS_KEY_ID - value: "minio_root_user" - - name: UNKEY_VAULT_S3_ACCESS_KEY_SECRET - value: "minio_root_password" + - name: UNKEY_VAULT_URL + value: "http://vault:8060" + - name: UNKEY_VAULT_TOKEN + value: "vault-test-token-123" # ClickHouse Proxy Service Configuration - name: UNKEY_CHPROXY_AUTH_TOKEN value: "chproxy-test-token-123" diff --git a/internal/services/analytics/service.go b/internal/services/analytics/service.go index 28af72e062..2b1ccc8d6e 100644 --- a/internal/services/analytics/service.go +++ b/internal/services/analytics/service.go @@ -23,7 +23,7 @@ type connectionManager struct { connectionCache cache.Cache[string, clickhouse.ClickHouse] database db.Database baseURL string - vault *vault.Service + vault vault.Client } // ConnectionManagerConfig contains configuration for the connection manager @@ -32,7 +32,7 @@ type ConnectionManagerConfig struct { Database db.Database Clock clock.Clock BaseURL string // e.g., "http://clickhouse:8123/default" or "clickhouse://clickhouse:9000/default" - Vault *vault.Service + Vault vault.Client } // NewConnectionManager creates a new connection manager diff --git a/pkg/testutil/containers/containers.go b/pkg/testutil/containers/containers.go index 918605e56a..1f5115bec2 100644 --- a/pkg/testutil/containers/containers.go +++ b/pkg/testutil/containers/containers.go @@ -178,6 +178,19 @@ func OTEL(t *testing.T) OTELConfig { } } +// Vault returns the URL and bearer token for the vault service in integration testing. +// +// The vault service runs on port 8060 and requires a bearer token for authentication. +// These values match the vault service configuration in docker-compose.yaml. +// +// Example usage: +// +// vaultURL, vaultToken := containers.Vault(t) +// client := vaultv1connect.NewVaultServiceClient(httpClient, vaultURL, ...) +func Vault(t *testing.T) (string, string) { + return "http://localhost:8060", "vault-test-token-123" +} + // Kafka returns Kafka broker addresses for integration testing. // // Returns broker addresses for connecting to the Kafka service running diff --git a/pkg/vault/BUILD.bazel b/pkg/vault/BUILD.bazel index 4242e7e011..2db18bbc32 100644 --- a/pkg/vault/BUILD.bazel +++ b/pkg/vault/BUILD.bazel @@ -3,26 +3,14 @@ load("@rules_go//go:def.bzl", "go_library") go_library( name = "vault", srcs = [ - "create_dek.go", - "decrypt.go", - "encrypt.go", - "reencrypt.go", - "roll_deks.go", - "service.go", + "client.go", + "connect_client.go", ], importpath = "github.com/unkeyed/unkey/pkg/vault", visibility = ["//visibility:public"], deps = [ "//gen/proto/vault/v1:vault", - "//pkg/cache", - "//pkg/cache/middleware", - "//pkg/clock", - "//pkg/encryption", - "//pkg/logger", - "//pkg/otel/tracing", - "//pkg/vault/keyring", - "//pkg/vault/storage", - "@io_opentelemetry_go_otel//attribute", - "@org_golang_google_protobuf//proto", + "//gen/proto/vault/v1/vaultv1connect", + "@com_connectrpc_connect//:connect", ], ) diff --git a/pkg/vault/client.go b/pkg/vault/client.go new file mode 100644 index 0000000000..e5ddfec2ce --- /dev/null +++ b/pkg/vault/client.go @@ -0,0 +1,14 @@ +package vault + +import ( + "context" + + vaultv1 "github.com/unkeyed/unkey/gen/proto/vault/v1" +) + +// Client defines the interface for vault encryption and decryption operations. +// [ConnectClient] implements this interface by wrapping a remote vault service. +type Client interface { + Encrypt(ctx context.Context, req *vaultv1.EncryptRequest) (*vaultv1.EncryptResponse, error) + Decrypt(ctx context.Context, req *vaultv1.DecryptRequest) (*vaultv1.DecryptResponse, error) +} diff --git a/pkg/vault/connect_client.go b/pkg/vault/connect_client.go new file mode 100644 index 0000000000..003e891b5d --- /dev/null +++ b/pkg/vault/connect_client.go @@ -0,0 +1,39 @@ +package vault + +import ( + "context" + + "connectrpc.com/connect" + vaultv1 "github.com/unkeyed/unkey/gen/proto/vault/v1" + "github.com/unkeyed/unkey/gen/proto/vault/v1/vaultv1connect" +) + +// Compile-time check that *ConnectClient implements Client. +var _ Client = (*ConnectClient)(nil) + +// ConnectClient adapts a [vaultv1connect.VaultServiceClient] to the [Client] interface, +// wrapping and unwrapping connect.Request/Response types. +type ConnectClient struct { + inner vaultv1connect.VaultServiceClient +} + +// NewConnectClient creates a new [ConnectClient] wrapping the given connect client. +func NewConnectClient(inner vaultv1connect.VaultServiceClient) *ConnectClient { + return &ConnectClient{inner: inner} +} + +func (c *ConnectClient) Encrypt(ctx context.Context, req *vaultv1.EncryptRequest) (*vaultv1.EncryptResponse, error) { + resp, err := c.inner.Encrypt(ctx, connect.NewRequest(req)) + if err != nil { + return nil, err + } + return resp.Msg, nil +} + +func (c *ConnectClient) Decrypt(ctx context.Context, req *vaultv1.DecryptRequest) (*vaultv1.DecryptResponse, error) { + resp, err := c.inner.Decrypt(ctx, connect.NewRequest(req)) + if err != nil { + return nil, err + } + return resp.Msg, nil +} diff --git a/pkg/vault/create_dek.go b/pkg/vault/create_dek.go deleted file mode 100644 index c45b1e56e8..0000000000 --- a/pkg/vault/create_dek.go +++ /dev/null @@ -1,18 +0,0 @@ -package vault - -import ( - "context" - - "github.com/unkeyed/unkey/pkg/otel/tracing" -) - -func (s *Service) CreateDEK(ctx context.Context, keyring string) (string, error) { - ctx, span := tracing.Start(ctx, "vault.CreateDEK") - defer span.End() - - key, err := s.keyring.CreateKey(ctx, keyring) - if err != nil { - return "", err - } - return key.GetId(), nil -} diff --git a/pkg/vault/decrypt.go b/pkg/vault/decrypt.go deleted file mode 100644 index 9dcaa3fde2..0000000000 --- a/pkg/vault/decrypt.go +++ /dev/null @@ -1,52 +0,0 @@ -package vault - -import ( - "context" - "encoding/base64" - "fmt" - - vaultv1 "github.com/unkeyed/unkey/gen/proto/vault/v1" - "github.com/unkeyed/unkey/pkg/cache" - "github.com/unkeyed/unkey/pkg/encryption" - "github.com/unkeyed/unkey/pkg/otel/tracing" - "google.golang.org/protobuf/proto" -) - -func (s *Service) Decrypt( - ctx context.Context, - req *vaultv1.DecryptRequest, -) (*vaultv1.DecryptResponse, error) { - ctx, span := tracing.Start(ctx, "vault.Decrypt") - defer span.End() - - b, err := base64.StdEncoding.DecodeString(req.GetEncrypted()) - if err != nil { - return nil, fmt.Errorf("failed to decode encrypted data: %w", err) - } - encrypted := vaultv1.Encrypted{} // nolint:exhaustruct - err = proto.Unmarshal(b, &encrypted) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal encrypted data: %w", err) - } - - cacheKey := fmt.Sprintf("%s-%s", req.GetKeyring(), encrypted.GetEncryptionKeyId()) - - dek, hit := s.keyCache.Get(ctx, cacheKey) - if hit == cache.Miss { - dek, err = s.keyring.GetKey(ctx, req.GetKeyring(), encrypted.GetEncryptionKeyId()) - if err != nil { - return nil, fmt.Errorf("failed to get dek in keyring %s: %w", req.GetKeyring(), err) - } - s.keyCache.Set(ctx, cacheKey, dek) - } - - plaintext, err := encryption.Decrypt(dek.GetKey(), encrypted.GetNonce(), encrypted.GetCiphertext()) - if err != nil { - return nil, fmt.Errorf("failed to decrypt ciphertext: %w", err) - } - - return &vaultv1.DecryptResponse{ - Plaintext: string(plaintext), - }, nil - -} diff --git a/pkg/vault/encrypt.go b/pkg/vault/encrypt.go deleted file mode 100644 index 7c782ec020..0000000000 --- a/pkg/vault/encrypt.go +++ /dev/null @@ -1,59 +0,0 @@ -package vault - -import ( - "context" - "encoding/base64" - "fmt" - "time" - - vaultv1 "github.com/unkeyed/unkey/gen/proto/vault/v1" - "github.com/unkeyed/unkey/pkg/cache" - "github.com/unkeyed/unkey/pkg/encryption" - "github.com/unkeyed/unkey/pkg/otel/tracing" - "go.opentelemetry.io/otel/attribute" - "google.golang.org/protobuf/proto" -) - -func (s *Service) Encrypt( - ctx context.Context, - req *vaultv1.EncryptRequest, -) (*vaultv1.EncryptResponse, error) { - ctx, span := tracing.Start(ctx, "vault.Encrypt") - defer span.End() - span.SetAttributes(attribute.String("keyring", req.GetKeyring())) - - cacheKey := fmt.Sprintf("%s-%s", req.GetKeyring(), LATEST) - - dek, hit := s.keyCache.Get(ctx, cacheKey) - if hit != cache.Hit { - var err error - dek, err = s.keyring.GetOrCreateKey(ctx, req.GetKeyring(), LATEST) - if err != nil { - return nil, fmt.Errorf("failed to get latest dek in keyring %s: %w", req.GetKeyring(), err) - } - s.keyCache.Set(ctx, cacheKey, dek) - } - - nonce, ciphertext, err := encryption.Encrypt(dek.GetKey(), []byte(req.GetData())) - if err != nil { - return nil, fmt.Errorf("failed to encrypt data: %w", err) - } - - encryptedData := &vaultv1.Encrypted{ - Algorithm: vaultv1.Algorithm_AES_256_GCM, - Nonce: nonce, - Ciphertext: ciphertext, - EncryptionKeyId: dek.GetId(), - Time: time.Now().UnixMilli(), - } - - b, err := proto.Marshal(encryptedData) - if err != nil { - return nil, fmt.Errorf("failed to marshal encrypted data: %w", err) - } - - return &vaultv1.EncryptResponse{ - Encrypted: base64.StdEncoding.EncodeToString(b), - KeyId: dek.GetId(), - }, nil -} diff --git a/pkg/vault/integration/BUILD.bazel b/pkg/vault/integration/BUILD.bazel deleted file mode 100644 index acf4209cc0..0000000000 --- a/pkg/vault/integration/BUILD.bazel +++ /dev/null @@ -1,21 +0,0 @@ -load("@rules_go//go:def.bzl", "go_test") - -go_test( - name = "integration_test", - size = "small", - srcs = [ - "coldstart_test.go", - "migrate_deks_test.go", - "reencryption_test.go", - "reusing_deks_test.go", - ], - deps = [ - "//gen/proto/vault/v1:vault", - "//pkg/testutil/containers", - "//pkg/uid", - "//pkg/vault", - "//pkg/vault/keys", - "//pkg/vault/storage", - "@com_github_stretchr_testify//require", - ], -) diff --git a/pkg/vault/integration/coldstart_test.go b/pkg/vault/integration/coldstart_test.go deleted file mode 100644 index 073fd1cb02..0000000000 --- a/pkg/vault/integration/coldstart_test.go +++ /dev/null @@ -1,88 +0,0 @@ -package integration_test - -import ( - "context" - "testing" - - "github.com/stretchr/testify/require" - vaultv1 "github.com/unkeyed/unkey/gen/proto/vault/v1" - "github.com/unkeyed/unkey/pkg/testutil/containers" - "github.com/unkeyed/unkey/pkg/uid" - "github.com/unkeyed/unkey/pkg/vault" - "github.com/unkeyed/unkey/pkg/vault/keys" - "github.com/unkeyed/unkey/pkg/vault/storage" -) - -// This scenario tests the cold start of the vault service. -// There are no keys in the storage and a few users are starting to use it - -func Test_ColdStart(t *testing.T) { - - s3 := containers.S3(t) - - storage, err := storage.NewS3(storage.S3Config{ - S3URL: s3.HostURL, - S3Bucket: "test", - S3AccessKeyID: s3.AccessKeyID, - S3AccessKeySecret: s3.AccessKeySecret, - }) - require.NoError(t, err) - - _, masterKey, err := keys.GenerateMasterKey() - require.NoError(t, err) - - v, err := vault.New(vault.Config{ - Storage: storage, - MasterKeys: []string{masterKey}, - }) - require.NoError(t, err) - - ctx := context.Background() - - aliceKeyRing := uid.New("alice") - bobKeyRing := uid.New("bob") - // Alice encrypts a secret - aliceData := "alice secret" - aliceEncryptionRes, err := v.Encrypt(ctx, &vaultv1.EncryptRequest{ - Keyring: aliceKeyRing, - Data: aliceData, - }) - require.NoError(t, err) - - // Bob encrypts a secret - bobData := "bob secret" - bobEncryptionRes, err := v.Encrypt(ctx, &vaultv1.EncryptRequest{ - Keyring: bobKeyRing, - Data: bobData, - }) - require.NoError(t, err) - - // Alice decrypts her secret - aliceDecryptionRes, err := v.Decrypt(ctx, &vaultv1.DecryptRequest{ - Keyring: aliceKeyRing, - Encrypted: aliceEncryptionRes.GetEncrypted(), - }) - require.NoError(t, err) - require.Equal(t, aliceData, aliceDecryptionRes.GetPlaintext()) - - // Bob reencrypts his secret - - _, err = v.CreateDEK(ctx, bobKeyRing) - require.NoError(t, err) - bobReencryptionRes, err := v.ReEncrypt(ctx, &vaultv1.ReEncryptRequest{ - Keyring: bobKeyRing, - Encrypted: bobEncryptionRes.GetEncrypted(), - }) - require.NoError(t, err) - - // Bob decrypts his secret - bobDecryptionRes, err := v.Decrypt(ctx, &vaultv1.DecryptRequest{ - Keyring: bobKeyRing, - Encrypted: bobReencryptionRes.GetEncrypted(), - }) - require.NoError(t, err) - require.Equal(t, bobData, bobDecryptionRes.GetPlaintext()) - // expect the key to be different - require.NotEqual(t, bobEncryptionRes.GetKeyId(), bobReencryptionRes.GetKeyId()) - -} diff --git a/pkg/vault/integration/migrate_deks_test.go b/pkg/vault/integration/migrate_deks_test.go deleted file mode 100644 index c558976af6..0000000000 --- a/pkg/vault/integration/migrate_deks_test.go +++ /dev/null @@ -1,105 +0,0 @@ -package integration_test - -import ( - "context" - "crypto/rand" - "testing" - "time" - - "fmt" - - "github.com/stretchr/testify/require" - vaultv1 "github.com/unkeyed/unkey/gen/proto/vault/v1" - "github.com/unkeyed/unkey/pkg/testutil/containers" - "github.com/unkeyed/unkey/pkg/uid" - "github.com/unkeyed/unkey/pkg/vault" - "github.com/unkeyed/unkey/pkg/vault/keys" - "github.com/unkeyed/unkey/pkg/vault/storage" -) - -// This scenario tests the re-encryption of a secret. -func TestMigrateDeks(t *testing.T) { - - data := make(map[string]string) - s3 := containers.S3(t) - - storage, err := storage.NewS3(storage.S3Config{ - S3URL: s3.HostURL, - S3Bucket: fmt.Sprintf("%d", time.Now().Unix()), - S3AccessKeyID: s3.AccessKeyID, - S3AccessKeySecret: s3.AccessKeySecret, - }) - require.NoError(t, err) - - _, masterKeyOld, err := keys.GenerateMasterKey() - require.NoError(t, err) - - v, err := vault.New(vault.Config{ - Storage: storage, - MasterKeys: []string{masterKeyOld}, - }) - require.NoError(t, err) - - ctx := context.Background() - - keyring := uid.New("test") - // Seed some DEKs - for range 10 { - - _, err = v.CreateDEK(ctx, keyring) - require.NoError(t, err) - - buf := make([]byte, 32) - _, err = rand.Read(buf) - d := string(buf) - require.NoError(t, err) - res, encryptErr := v.Encrypt(ctx, &vaultv1.EncryptRequest{ - Keyring: keyring, - Data: d, - }) - require.NoError(t, encryptErr) - data[d] = res.GetEncrypted() - } - - // Simulate Restart - - _, masterKeyNew, err := keys.GenerateMasterKey() - require.NoError(t, err) - - v, err = vault.New(vault.Config{ - Storage: storage, - MasterKeys: []string{masterKeyOld, masterKeyNew}, - }) - require.NoError(t, err) - - err = v.RollDeks(ctx) - require.NoError(t, err) - - // Check each piece of data can be decrypted - for d, e := range data { - res, decryptErr := v.Decrypt(ctx, &vaultv1.DecryptRequest{ - Keyring: keyring, - Encrypted: e, - }) - require.NoError(t, decryptErr) - require.Equal(t, d, res.GetPlaintext()) - } - // Simulate another restart, removing the old master key - - v, err = vault.New(vault.Config{ - Storage: storage, - MasterKeys: []string{masterKeyNew}, - }) - require.NoError(t, err) - - // Check each piece of data can be decrypted - for d, e := range data { - res, err := v.Decrypt(ctx, &vaultv1.DecryptRequest{ - Keyring: keyring, - Encrypted: e, - }) - require.NoError(t, err) - require.Equal(t, d, res.GetPlaintext()) - } - -} diff --git a/pkg/vault/integration/reencryption_test.go b/pkg/vault/integration/reencryption_test.go deleted file mode 100644 index 6df5fe2a33..0000000000 --- a/pkg/vault/integration/reencryption_test.go +++ /dev/null @@ -1,84 +0,0 @@ -package integration_test - -import ( - "context" - "crypto/rand" - "fmt" - "math" - "testing" - - "github.com/stretchr/testify/require" - vaultv1 "github.com/unkeyed/unkey/gen/proto/vault/v1" - "github.com/unkeyed/unkey/pkg/testutil/containers" - "github.com/unkeyed/unkey/pkg/uid" - "github.com/unkeyed/unkey/pkg/vault" - "github.com/unkeyed/unkey/pkg/vault/keys" - "github.com/unkeyed/unkey/pkg/vault/storage" -) - -// This scenario tests the re-encryption of a secret. -func TestReEncrypt(t *testing.T) { - - s3 := containers.S3(t) - - storage, err := storage.NewS3(storage.S3Config{ - S3URL: s3.HostURL, - S3Bucket: "vault", - S3AccessKeyID: s3.AccessKeyID, - S3AccessKeySecret: s3.AccessKeySecret, - }) - require.NoError(t, err) - - _, masterKey, err := keys.GenerateMasterKey() - require.NoError(t, err) - - v, err := vault.New(vault.Config{ - Storage: storage, - MasterKeys: []string{masterKey}, - }) - require.NoError(t, err) - - ctx := context.Background() - - for i := 1; i < 9; i++ { - - dataSize := int(math.Pow(8, float64(i))) - t.Run(fmt.Sprintf("with %d bytes", dataSize), func(t *testing.T) { - - keyring := uid.New("test") - buf := make([]byte, dataSize) - _, err := rand.Read(buf) - require.NoError(t, err) - - data := string(buf) - - enc, err := v.Encrypt(ctx, &vaultv1.EncryptRequest{ - Keyring: keyring, - Data: data, - }) - require.NoError(t, err) - - deks := []string{} - for range 10 { - dekID, createDekErr := v.CreateDEK(ctx, keyring) - require.NoError(t, createDekErr) - require.NotContains(t, deks, dekID) - deks = append(deks, dekID) - _, err = v.ReEncrypt(ctx, &vaultv1.ReEncryptRequest{ - Keyring: keyring, - Encrypted: enc.GetEncrypted(), - }) - require.NoError(t, err) - } - - dec, err := v.Decrypt(ctx, &vaultv1.DecryptRequest{ - Keyring: keyring, - Encrypted: enc.GetEncrypted(), - }) - require.NoError(t, err) - require.Equal(t, data, dec.GetPlaintext()) - }) - - } - -} diff --git a/pkg/vault/integration/reusing_deks_test.go b/pkg/vault/integration/reusing_deks_test.go deleted file mode 100644 index 61d4dae684..0000000000 --- a/pkg/vault/integration/reusing_deks_test.go +++ /dev/null @@ -1,95 +0,0 @@ -package integration_test - -import ( - "context" - "testing" - - "fmt" - "time" - - "github.com/stretchr/testify/require" - vaultv1 "github.com/unkeyed/unkey/gen/proto/vault/v1" - "github.com/unkeyed/unkey/pkg/testutil/containers" - "github.com/unkeyed/unkey/pkg/uid" - "github.com/unkeyed/unkey/pkg/vault" - "github.com/unkeyed/unkey/pkg/vault/keys" - "github.com/unkeyed/unkey/pkg/vault/storage" -) - -// When encrypting multiple secrets with the same keyring, the same DEK should be reused for all of them. -func TestReuseDEKsForSameKeyring(t *testing.T) { - - s3 := containers.S3(t) - - storage, err := storage.NewS3(storage.S3Config{ - S3URL: s3.HostURL, - S3Bucket: fmt.Sprintf("%d", time.Now().UnixMilli()), - S3AccessKeyID: s3.AccessKeyID, - S3AccessKeySecret: s3.AccessKeySecret, - }) - require.NoError(t, err) - - _, masterKey, err := keys.GenerateMasterKey() - require.NoError(t, err) - - v, err := vault.New(vault.Config{ - Storage: storage, - MasterKeys: []string{masterKey}, - }) - require.NoError(t, err) - - ctx := context.Background() - - deks := map[string]bool{} - - for range 10 { - res, encryptErr := v.Encrypt(ctx, &vaultv1.EncryptRequest{ - Keyring: "keyring", - Data: uid.New(uid.TestPrefix), - }) - require.NoError(t, encryptErr) - deks[res.GetKeyId()] = true - } - - require.Len(t, deks, 1) - -} - -// When encrypting multiple secrets with different keyrings, a different DEK should be used for each keyring. -func TestIndividualDEKsPerKeyring(t *testing.T) { - - s3 := containers.S3(t) - - storage, err := storage.NewS3(storage.S3Config{ - S3URL: s3.HostURL, - S3Bucket: fmt.Sprintf("%d", time.Now().UnixMilli()), - S3AccessKeyID: s3.AccessKeyID, - S3AccessKeySecret: s3.AccessKeySecret, - }) - require.NoError(t, err) - - _, masterKey, err := keys.GenerateMasterKey() - require.NoError(t, err) - - v, err := vault.New(vault.Config{ - Storage: storage, - MasterKeys: []string{masterKey}, - }) - require.NoError(t, err) - - ctx := context.Background() - - deks := map[string]bool{} - - for range 10 { - res, encryptErr := v.Encrypt(ctx, &vaultv1.EncryptRequest{ - Keyring: uid.New(uid.TestPrefix), - Data: uid.New(uid.TestPrefix), - }) - require.NoError(t, encryptErr) - deks[res.GetKeyId()] = true - } - - require.Len(t, deks, 10) - -} diff --git a/pkg/vault/keyring/BUILD.bazel b/pkg/vault/keyring/BUILD.bazel deleted file mode 100644 index 536bad9fd5..0000000000 --- a/pkg/vault/keyring/BUILD.bazel +++ /dev/null @@ -1,27 +0,0 @@ -load("@rules_go//go:def.bzl", "go_library") - -go_library( - name = "keyring", - srcs = [ - "create_key.go", - "decode_and_decrypt_key.go", - "encrypt_and_encode_key.go", - "get_key.go", - "get_latest_key.go", - "get_or_create_key.go", - "keyring.go", - "roll_keys.go", - ], - importpath = "github.com/unkeyed/unkey/pkg/vault/keyring", - visibility = ["//visibility:public"], - deps = [ - "//gen/proto/vault/v1:vault", - "//pkg/encryption", - "//pkg/logger", - "//pkg/otel/tracing", - "//pkg/vault/keys", - "//pkg/vault/storage", - "@io_opentelemetry_go_otel//attribute", - "@org_golang_google_protobuf//proto", - ], -) diff --git a/pkg/vault/keyring/create_key.go b/pkg/vault/keyring/create_key.go deleted file mode 100644 index 00ae8ba5cf..0000000000 --- a/pkg/vault/keyring/create_key.go +++ /dev/null @@ -1,42 +0,0 @@ -package keyring - -import ( - "context" - "fmt" - "time" - - vaultv1 "github.com/unkeyed/unkey/gen/proto/vault/v1" - "github.com/unkeyed/unkey/pkg/otel/tracing" - "github.com/unkeyed/unkey/pkg/vault/keys" -) - -func (k *Keyring) CreateKey(ctx context.Context, ringID string) (*vaultv1.DataEncryptionKey, error) { - ctx, span := tracing.Start(ctx, "keyring.CreateKey") - defer span.End() - keyId, key, err := keys.GenerateKey("dek") - if err != nil { - return nil, fmt.Errorf("failed to generate key: %w", err) - } - - dek := &vaultv1.DataEncryptionKey{ - Id: keyId, - Key: key, - CreatedAt: time.Now().UnixMilli(), - } - - b, err := k.EncryptAndEncodeKey(ctx, dek) - if err != nil { - return nil, fmt.Errorf("failed to encrypt and encode dek: %w", err) - } - - err = k.store.PutObject(ctx, k.buildLookupKey(ringID, dek.GetId()), b) - if err != nil { - return nil, fmt.Errorf("failed to put encrypted dek: %w", err) - } - err = k.store.PutObject(ctx, k.buildLookupKey(ringID, "LATEST"), b) - if err != nil { - return nil, fmt.Errorf("failed to put encrypted dek: %w", err) - } - - return dek, nil -} diff --git a/pkg/vault/keyring/decode_and_decrypt_key.go b/pkg/vault/keyring/decode_and_decrypt_key.go deleted file mode 100644 index 58e8b88b71..0000000000 --- a/pkg/vault/keyring/decode_and_decrypt_key.go +++ /dev/null @@ -1,44 +0,0 @@ -package keyring - -import ( - "context" - "fmt" - - vaultv1 "github.com/unkeyed/unkey/gen/proto/vault/v1" - "github.com/unkeyed/unkey/pkg/encryption" - "github.com/unkeyed/unkey/pkg/otel/tracing" - "google.golang.org/protobuf/proto" -) - -func (k *Keyring) DecodeAndDecryptKey(ctx context.Context, b []byte) (*vaultv1.DataEncryptionKey, string, error) { - _, span := tracing.Start(ctx, "keyring.DecodeAndDecryptKey") - defer span.End() - encrypted := &vaultv1.EncryptedDataEncryptionKey{} // nolint:exhaustruct - err := proto.Unmarshal(b, encrypted) - if err != nil { - tracing.RecordError(span, err) - return nil, "", fmt.Errorf("failed to unmarshal encrypted dek: %w", err) - } - - kek, ok := k.decryptionKeys[encrypted.GetEncrypted().GetEncryptionKeyId()] - if !ok { - err = fmt.Errorf("no kek found for key id: %s", encrypted.GetEncrypted().GetEncryptionKeyId()) - tracing.RecordError(span, err) - return nil, "", err - } - - plaintext, err := encryption.Decrypt(kek.GetKey(), encrypted.GetEncrypted().GetNonce(), encrypted.GetEncrypted().GetCiphertext()) - if err != nil { - tracing.RecordError(span, err) - return nil, "", fmt.Errorf("failed to decrypt ciphertext: %w", err) - } - - dek := &vaultv1.DataEncryptionKey{} // nolint:exhaustruct - err = proto.Unmarshal(plaintext, dek) - if err != nil { - tracing.RecordError(span, err) - return nil, "", fmt.Errorf("failed to unmarshal dek: %w", err) - } - return dek, encrypted.GetEncrypted().GetEncryptionKeyId(), nil - -} diff --git a/pkg/vault/keyring/encrypt_and_encode_key.go b/pkg/vault/keyring/encrypt_and_encode_key.go deleted file mode 100644 index 23ce654214..0000000000 --- a/pkg/vault/keyring/encrypt_and_encode_key.go +++ /dev/null @@ -1,44 +0,0 @@ -package keyring - -import ( - "context" - "fmt" - "time" - - vaultv1 "github.com/unkeyed/unkey/gen/proto/vault/v1" - "github.com/unkeyed/unkey/pkg/encryption" - "github.com/unkeyed/unkey/pkg/otel/tracing" - "google.golang.org/protobuf/proto" -) - -func (k *Keyring) EncryptAndEncodeKey(ctx context.Context, dek *vaultv1.DataEncryptionKey) ([]byte, error) { - _, span := tracing.Start(ctx, "keyring.EncryptAndEncodeKey") - defer span.End() - b, err := proto.Marshal(dek) - if err != nil { - return nil, fmt.Errorf("failed to marshal dek: %w", err) - } - - nonce, ciphertext, err := encryption.Encrypt(k.encryptionKey.GetKey(), b) - if err != nil { - return nil, fmt.Errorf("failed to encrypt dek: %w", err) - } - - encryptedDek := &vaultv1.EncryptedDataEncryptionKey{ - Id: dek.GetId(), - CreatedAt: dek.GetCreatedAt(), - Encrypted: &vaultv1.Encrypted{ - Algorithm: vaultv1.Algorithm_AES_256_GCM, - Nonce: nonce, - Ciphertext: ciphertext, - EncryptionKeyId: k.encryptionKey.GetId(), - Time: time.Now().UnixMilli(), - }, - } - - b, err = proto.Marshal(encryptedDek) - if err != nil { - return nil, fmt.Errorf("failed to marshal encrypted dek: %w", err) - } - return b, nil -} diff --git a/pkg/vault/keyring/get_key.go b/pkg/vault/keyring/get_key.go deleted file mode 100644 index 1c3cef9b35..0000000000 --- a/pkg/vault/keyring/get_key.go +++ /dev/null @@ -1,37 +0,0 @@ -package keyring - -import ( - "context" - "fmt" - - vaultv1 "github.com/unkeyed/unkey/gen/proto/vault/v1" - "github.com/unkeyed/unkey/pkg/otel/tracing" - "github.com/unkeyed/unkey/pkg/vault/storage" - "go.opentelemetry.io/otel/attribute" -) - -func (k *Keyring) GetKey(ctx context.Context, ringID, keyID string) (*vaultv1.DataEncryptionKey, error) { - ctx, span := tracing.Start(ctx, "keyring.GetKey") - defer span.End() - - lookupKey := k.buildLookupKey(ringID, keyID) - span.SetAttributes(attribute.String("lookupKey", lookupKey)) - - b, found, err := k.store.GetObject(ctx, lookupKey) - span.SetAttributes(attribute.Bool("found", found)) - if err != nil { - tracing.RecordError(span, err) - return nil, fmt.Errorf("failed to get object: %w", err) - - } - if !found { - return nil, storage.ErrObjectNotFound - } - - dek, _, err := k.DecodeAndDecryptKey(ctx, b) - if err != nil { - tracing.RecordError(span, err) - return nil, fmt.Errorf("failed to decode and decrypt key: %w", err) - } - return dek, nil -} diff --git a/pkg/vault/keyring/get_latest_key.go b/pkg/vault/keyring/get_latest_key.go deleted file mode 100644 index 0d5d213bbf..0000000000 --- a/pkg/vault/keyring/get_latest_key.go +++ /dev/null @@ -1,29 +0,0 @@ -package keyring - -import ( - "context" - "fmt" - - vaultv1 "github.com/unkeyed/unkey/gen/proto/vault/v1" - "github.com/unkeyed/unkey/pkg/otel/tracing" - - "github.com/unkeyed/unkey/pkg/vault/storage" -) - -// GetLatestKey returns the latest key from the keyring. If no key is found, it creates a new key. -func (k *Keyring) GetLatestKey(ctx context.Context, ringID string) (*vaultv1.DataEncryptionKey, error) { - ctx, span := tracing.Start(ctx, "keyring.GetLatestKey") - defer span.End() - dek, err := k.GetKey(ctx, ringID, "LATEST") - - if err == nil { - return dek, nil - } - - if err != storage.ErrObjectNotFound { - tracing.RecordError(span, err) - return nil, fmt.Errorf("failed to get key: %w", err) - } - - return k.CreateKey(ctx, ringID) -} diff --git a/pkg/vault/keyring/get_or_create_key.go b/pkg/vault/keyring/get_or_create_key.go deleted file mode 100644 index 72ce6b9d1e..0000000000 --- a/pkg/vault/keyring/get_or_create_key.go +++ /dev/null @@ -1,31 +0,0 @@ -package keyring - -import ( - "context" - "errors" - "fmt" - - vaultv1 "github.com/unkeyed/unkey/gen/proto/vault/v1" - "github.com/unkeyed/unkey/pkg/otel/tracing" - "github.com/unkeyed/unkey/pkg/vault/storage" - "go.opentelemetry.io/otel/attribute" -) - -func (k *Keyring) GetOrCreateKey(ctx context.Context, ringID, keyID string) (*vaultv1.DataEncryptionKey, error) { - ctx, span := tracing.Start(ctx, "keyring.GetOrCreateKey") - defer span.End() - span.SetAttributes(attribute.String("ringID", ringID), attribute.String("keyID", keyID)) - dek, err := k.GetKey(ctx, ringID, keyID) - if err == nil { - return dek, nil - } - - if errors.Is(err, storage.ErrObjectNotFound) { - return k.CreateKey(ctx, ringID) - } - - tracing.RecordError(span, err) - - return nil, fmt.Errorf("failed to get key: %w", err) - -} diff --git a/pkg/vault/keyring/keyring.go b/pkg/vault/keyring/keyring.go deleted file mode 100644 index 1890037ac5..0000000000 --- a/pkg/vault/keyring/keyring.go +++ /dev/null @@ -1,37 +0,0 @@ -package keyring - -import ( - "fmt" - - vaultv1 "github.com/unkeyed/unkey/gen/proto/vault/v1" - "github.com/unkeyed/unkey/pkg/vault/storage" -) - -type Keyring struct { - store storage.Storage - - // any of these can be used for decryption - decryptionKeys map[string]*vaultv1.KeyEncryptionKey - encryptionKey *vaultv1.KeyEncryptionKey -} - -type Config struct { - Store storage.Storage - - DecryptionKeys map[string]*vaultv1.KeyEncryptionKey - EncryptionKey *vaultv1.KeyEncryptionKey -} - -func New(config Config) (*Keyring, error) { - - return &Keyring{ - store: config.Store, - encryptionKey: config.EncryptionKey, - decryptionKeys: config.DecryptionKeys, - }, nil -} - -// The storage layer doesn't know about keyrings, so we need to prefix the key with the keyring id -func (k *Keyring) buildLookupKey(ringID, dekID string) string { - return fmt.Sprintf("keyring/%s/%s", ringID, dekID) -} diff --git a/pkg/vault/keyring/roll_keys.go b/pkg/vault/keyring/roll_keys.go deleted file mode 100644 index 422255a12c..0000000000 --- a/pkg/vault/keyring/roll_keys.go +++ /dev/null @@ -1,51 +0,0 @@ -package keyring - -import ( - "context" - "fmt" - - "github.com/unkeyed/unkey/pkg/logger" - "github.com/unkeyed/unkey/pkg/otel/tracing" - "github.com/unkeyed/unkey/pkg/vault/storage" -) - -func (k *Keyring) RollKeys(ctx context.Context, ringID string) error { - ctx, span := tracing.Start(ctx, "keyring.RollKeys") - defer span.End() - lookupKeys, err := k.store.ListObjectKeys(ctx, k.buildLookupKey(ringID, "dek_")) - if err != nil { - return fmt.Errorf("failed to list keys: %w", err) - } - - for _, objectKey := range lookupKeys { - b, found, err := k.store.GetObject(ctx, objectKey) - if err != nil { - return fmt.Errorf("failed to get object: %w", err) - } - if !found { - return storage.ErrObjectNotFound - } - - dek, encryptionKeyId, err := k.DecodeAndDecryptKey(ctx, b) - if err != nil { - return fmt.Errorf("failed to decode and decrypt key: %w", err) - } - if encryptionKeyId == k.encryptionKey.GetId() { - logger.Info("key already encrypted with latest kek", - "keyId", dek.GetId(), - ) - continue - } - reencrypted, err := k.EncryptAndEncodeKey(ctx, dek) - if err != nil { - return fmt.Errorf("failed to re-encrypt key: %w", err) - } - err = k.store.PutObject(ctx, objectKey, reencrypted) - if err != nil { - return fmt.Errorf("failed to put re-encrypted key: %w", err) - } - } - - return nil - -} diff --git a/pkg/vault/reencrypt.go b/pkg/vault/reencrypt.go deleted file mode 100644 index 5ef9b8dae0..0000000000 --- a/pkg/vault/reencrypt.go +++ /dev/null @@ -1,41 +0,0 @@ -package vault - -import ( - "context" - "fmt" - - vaultv1 "github.com/unkeyed/unkey/gen/proto/vault/v1" - "github.com/unkeyed/unkey/pkg/logger" - "github.com/unkeyed/unkey/pkg/otel/tracing" -) - -func (s *Service) ReEncrypt(ctx context.Context, req *vaultv1.ReEncryptRequest) (*vaultv1.ReEncryptResponse, error) { - ctx, span := tracing.Start(ctx, "vault.ReEncrypt") - defer span.End() - logger.Info("reencrypting", - "keyring", req.GetKeyring(), - ) - - decrypted, err := s.Decrypt(ctx, &vaultv1.DecryptRequest{ - Keyring: req.GetKeyring(), - Encrypted: req.GetEncrypted(), - }) - if err != nil { - return nil, fmt.Errorf("failed to decrypt: %w", err) - } - - s.keyCache.Clear(ctx) - - encrypted, err := s.Encrypt(ctx, &vaultv1.EncryptRequest{ - Keyring: req.GetKeyring(), - Data: decrypted.GetPlaintext(), - }) - if err != nil { - return nil, fmt.Errorf("failed to encrypt: %w", err) - } - return &vaultv1.ReEncryptResponse{ - Encrypted: encrypted.GetEncrypted(), - KeyId: encrypted.GetKeyId(), - }, nil - -} diff --git a/pkg/vault/roll_deks.go b/pkg/vault/roll_deks.go deleted file mode 100644 index 0ff180813c..0000000000 --- a/pkg/vault/roll_deks.go +++ /dev/null @@ -1,49 +0,0 @@ -package vault - -import ( - "context" - "fmt" - - "github.com/unkeyed/unkey/pkg/logger" - "github.com/unkeyed/unkey/pkg/otel/tracing" - "github.com/unkeyed/unkey/pkg/vault/storage" -) - -func (s *Service) RollDeks(ctx context.Context) error { - ctx, span := tracing.Start(ctx, "vault.RollDeks") - defer span.End() - lookupKeys, err := s.storage.ListObjectKeys(ctx, "keyring/") - if err != nil { - return fmt.Errorf("failed to list keys: %w", err) - } - - for _, objectKey := range lookupKeys { - b, found, err := s.storage.GetObject(ctx, objectKey) - if err != nil { - return fmt.Errorf("failed to get object: %w", err) - } - if !found { - return storage.ErrObjectNotFound - } - dek, kekID, err := s.keyring.DecodeAndDecryptKey(ctx, b) - if err != nil { - return fmt.Errorf("failed to decode and decrypt key: %w", err) - } - if kekID == s.encryptionKey.GetId() { - logger.Info("key already encrypted with latest kek", - "dekId", dek.GetId(), - ) - continue - } - reencrypted, err := s.keyring.EncryptAndEncodeKey(ctx, dek) - if err != nil { - return fmt.Errorf("failed to re-encrypt key: %w", err) - } - err = s.storage.PutObject(ctx, objectKey, reencrypted) - if err != nil { - return fmt.Errorf("failed to put re-encrypted key: %w", err) - } - } - - return nil -} diff --git a/pkg/vault/service.go b/pkg/vault/service.go deleted file mode 100644 index 4eb606c214..0000000000 --- a/pkg/vault/service.go +++ /dev/null @@ -1,109 +0,0 @@ -package vault - -import ( - "encoding/base64" - "fmt" - "time" - - vaultv1 "github.com/unkeyed/unkey/gen/proto/vault/v1" - "github.com/unkeyed/unkey/pkg/cache" - cacheMiddleware "github.com/unkeyed/unkey/pkg/cache/middleware" - "github.com/unkeyed/unkey/pkg/clock" - "github.com/unkeyed/unkey/pkg/vault/keyring" - "github.com/unkeyed/unkey/pkg/vault/storage" - "google.golang.org/protobuf/proto" -) - -// LATEST is a version identifier that refers to the most recent version of an -// encrypted value. Use this when you want to decrypt using the current key -// without specifying an explicit version number. -const LATEST = "LATEST" - -// Service provides encryption and decryption operations using a hierarchical -// key management scheme. It manages data encryption keys (DEKs) which are -// themselves encrypted by key encryption keys (KEKs/master keys). The service -// caches DEKs to reduce storage lookups and handles key rotation transparently. -type Service struct { - keyCache cache.Cache[string, *vaultv1.DataEncryptionKey] - - storage storage.Storage - - decryptionKeys map[string]*vaultv1.KeyEncryptionKey - encryptionKey *vaultv1.KeyEncryptionKey - - keyring *keyring.Keyring -} - -// Config holds configuration for creating a new vault [Service]. -type Config struct { - Storage storage.Storage - MasterKeys []string -} - -// New creates a new vault [Service] with the provided configuration. The last -// key in [Config.MasterKeys] is used for encryption, while all keys can be used -// for decryption, enabling seamless key rotation. -func New(cfg Config) (*Service, error) { - - encryptionKey, decryptionKeys, err := loadMasterKeys(cfg.MasterKeys) - if err != nil { - return nil, fmt.Errorf("unable to load master keys: %w", err) - - } - - kr, err := keyring.New(keyring.Config{ - Store: cfg.Storage, - DecryptionKeys: decryptionKeys, - EncryptionKey: encryptionKey, - }) - if err != nil { - return nil, fmt.Errorf("failed to create keyring: %w", err) - } - - cache, err := cache.New(cache.Config[string, *vaultv1.DataEncryptionKey]{ - Fresh: time.Hour, - Stale: 24 * time.Hour, - MaxSize: 10000, - Resource: "data_encryption_key", - Clock: clock.New(), - }) - if err != nil { - return nil, fmt.Errorf("failed to create cache: %w", err) - } - - return &Service{ - storage: cfg.Storage, - keyCache: cacheMiddleware.WithTracing(cache), - decryptionKeys: decryptionKeys, - - encryptionKey: encryptionKey, - keyring: kr, - }, nil -} - -func loadMasterKeys(masterKeys []string) (*vaultv1.KeyEncryptionKey, map[string]*vaultv1.KeyEncryptionKey, error) { - if len(masterKeys) == 0 { - return nil, nil, fmt.Errorf("no master keys provided") - } - encryptionKey := &vaultv1.KeyEncryptionKey{} // nolint:exhaustruct - decryptionKeys := make(map[string]*vaultv1.KeyEncryptionKey) - - for _, mk := range masterKeys { - kek := &vaultv1.KeyEncryptionKey{} // nolint:exhaustruct - b, err := base64.StdEncoding.DecodeString(mk) - if err != nil { - return nil, nil, fmt.Errorf("failed to decode master key: %w", err) - } - - err = proto.Unmarshal(b, kek) - if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal master key: %w", err) - } - - decryptionKeys[kek.GetId()] = kek - // this way, the last key in the list is used for encryption - encryptionKey = kek - - } - return encryptionKey, decryptionKeys, nil -} diff --git a/pkg/vault/storage/BUILD.bazel b/pkg/vault/storage/BUILD.bazel deleted file mode 100644 index 7c7bdbcb38..0000000000 --- a/pkg/vault/storage/BUILD.bazel +++ /dev/null @@ -1,20 +0,0 @@ -load("@rules_go//go:def.bzl", "go_library") - -go_library( - name = "storage", - srcs = [ - "interface.go", - "memory.go", - "s3.go", - ], - importpath = "github.com/unkeyed/unkey/pkg/vault/storage", - visibility = ["//visibility:public"], - deps = [ - "//pkg/fault", - "//pkg/logger", - "@com_github_aws_aws_sdk_go_v2//aws", - "@com_github_aws_aws_sdk_go_v2_config//:config", - "@com_github_aws_aws_sdk_go_v2_credentials//:credentials", - "@com_github_aws_aws_sdk_go_v2_service_s3//:s3", - ], -) diff --git a/pkg/vault/storage/interface.go b/pkg/vault/storage/interface.go deleted file mode 100644 index 98cb6a2d61..0000000000 --- a/pkg/vault/storage/interface.go +++ /dev/null @@ -1,30 +0,0 @@ -package storage - -import ( - "context" - "errors" - "time" -) - -var ErrObjectNotFound = errors.New("object not found") - -type GetObjectOptions struct { - IfUnModifiedSince time.Time -} - -type Storage interface { - // PutObject stores the object data for the given key - PutObject(ctx context.Context, key string, object []byte) error - - // GetObject returns the object data for the given key - GetObject(ctx context.Context, key string) ([]byte, bool, error) - - // ListObjectKeys returns a list of object keys that match the given prefix - ListObjectKeys(ctx context.Context, prefix string) ([]string, error) - - // Key returns the object key for the given shard and version - Key(shard string, dekID string) string - - // Latest returns the object key for the latest version of the given workspace - Latest(shard string) string -} diff --git a/pkg/vault/storage/memory.go b/pkg/vault/storage/memory.go deleted file mode 100644 index de1abcb6b2..0000000000 --- a/pkg/vault/storage/memory.go +++ /dev/null @@ -1,64 +0,0 @@ -package storage - -import ( - "context" - "fmt" - "strings" - "sync" -) - -// memory is an in-memory storage implementation for testing purposes. -type memory struct { - mu sync.RWMutex - data map[string][]byte -} - -func NewMemory() (Storage, error) { - return &memory{ - data: make(map[string][]byte), - mu: sync.RWMutex{}, - }, nil -} - -func (s *memory) Key(workspaceId string, dekID string) string { - return fmt.Sprintf("%s/%s", workspaceId, dekID) -} - -func (s *memory) Latest(workspaceId string) string { - return s.Key(workspaceId, "LATEST") -} - -func (s *memory) PutObject(ctx context.Context, key string, b []byte) error { - s.mu.Lock() - defer s.mu.Unlock() - - s.data[key] = b - return nil -} - -func (s *memory) GetObject(ctx context.Context, key string) ([]byte, bool, error) { - s.mu.RLock() - defer s.mu.RUnlock() - - b, ok := s.data[key] - if !ok { - return nil, false, nil - } - - return b, true, nil -} - -func (s *memory) ListObjectKeys(ctx context.Context, prefix string) ([]string, error) { - s.mu.RLock() - defer s.mu.RUnlock() - keys := []string{} - for key := range s.data { - if prefix == "" || !strings.HasPrefix(key, prefix) { - continue - } - - keys = append(keys, key) - - } - return keys, nil -} diff --git a/pkg/vault/storage/middleware/BUILD.bazel b/pkg/vault/storage/middleware/BUILD.bazel deleted file mode 100644 index 29d86b1993..0000000000 --- a/pkg/vault/storage/middleware/BUILD.bazel +++ /dev/null @@ -1,14 +0,0 @@ -load("@rules_go//go:def.bzl", "go_library") - -go_library( - name = "middleware", - srcs = ["tracing.go"], - importpath = "github.com/unkeyed/unkey/pkg/vault/storage/middleware", - visibility = ["//visibility:public"], - deps = [ - "//pkg/otel/tracing", - "//pkg/vault/storage", - "@io_opentelemetry_go_otel//attribute", - "@io_opentelemetry_go_otel//codes", - ], -) diff --git a/pkg/vault/storage/middleware/tracing.go b/pkg/vault/storage/middleware/tracing.go deleted file mode 100644 index 9bd44e358d..0000000000 --- a/pkg/vault/storage/middleware/tracing.go +++ /dev/null @@ -1,65 +0,0 @@ -package middleware - -import ( - "context" - "fmt" - - "github.com/unkeyed/unkey/pkg/otel/tracing" - "github.com/unkeyed/unkey/pkg/vault/storage" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/codes" -) - -type tracingMiddleware struct { - name string - next storage.Storage -} - -func WithTracing(name string, next storage.Storage) storage.Storage { - return &tracingMiddleware{ - name: name, - next: next, - } -} - -func (tm *tracingMiddleware) PutObject(ctx context.Context, key string, object []byte) error { - ctx, span := tracing.Start(ctx, fmt.Sprintf("storage.%s.PutObject", tm.name)) - defer span.End() - span.SetAttributes(attribute.String("key", key)) - err := tm.next.PutObject(ctx, key, object) - if err != nil { - span.SetStatus(codes.Error, err.Error()) - } - return err -} - -func (tm *tracingMiddleware) GetObject(ctx context.Context, key string) ([]byte, bool, error) { - ctx, span := tracing.Start(ctx, fmt.Sprintf("storage.%s.GetObject", tm.name)) - defer span.End() - span.SetAttributes(attribute.String("key", key)) - object, found, err := tm.next.GetObject(ctx, key) - span.SetAttributes(attribute.Bool("found", found)) - if err != nil { - span.SetStatus(codes.Error, err.Error()) - } - return object, found, err -} - -func (tm *tracingMiddleware) ListObjectKeys(ctx context.Context, prefix string) ([]string, error) { - ctx, span := tracing.Start(ctx, fmt.Sprintf("storage.%s.ListObjectKeys", tm.name)) - defer span.End() - span.SetAttributes(attribute.String("prefix", prefix)) - keys, err := tm.next.ListObjectKeys(ctx, prefix) - if err != nil { - span.SetStatus(codes.Error, err.Error()) - } - return keys, err -} - -func (tm *tracingMiddleware) Key(shard string, dekID string) string { - return tm.next.Key(shard, dekID) -} - -func (tm *tracingMiddleware) Latest(shard string) string { - return tm.next.Latest(shard) -} diff --git a/pkg/vault/storage/s3.go b/pkg/vault/storage/s3.go deleted file mode 100644 index 3f2b350778..0000000000 --- a/pkg/vault/storage/s3.go +++ /dev/null @@ -1,125 +0,0 @@ -package storage - -import ( - "bytes" - "context" - "fmt" - "io" - "strings" - - "github.com/aws/aws-sdk-go-v2/aws" - awsConfig "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/credentials" - awsS3 "github.com/aws/aws-sdk-go-v2/service/s3" - - "github.com/unkeyed/unkey/pkg/fault" - "github.com/unkeyed/unkey/pkg/logger" -) - -type s3 struct { - client *awsS3.Client - config S3Config -} - -type S3Config struct { - S3URL string - S3Bucket string - S3AccessKeyID string - S3AccessKeySecret string -} - -func NewS3(config S3Config) (Storage, error) { - logger.Info("using s3 storage") - - // nolint:staticcheck - r2Resolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...any) (aws.Endpoint, error) { - // nolint:staticcheck - return aws.Endpoint{ - URL: config.S3URL, - HostnameImmutable: true, - }, nil - }) - - cfg, err := awsConfig.LoadDefaultConfig(context.Background(), - awsConfig.WithEndpointResolverWithOptions(r2Resolver), // nolint:staticcheck - awsConfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(config.S3AccessKeyID, config.S3AccessKeySecret, "")), - awsConfig.WithRegion("auto"), - awsConfig.WithRetryMode(aws.RetryModeStandard), - awsConfig.WithRetryMaxAttempts(3), - ) - if err != nil { - return nil, fault.Wrap(err, fault.Internal("failed to load aws config"), fault.Public("failed to load aws config")) - } - - client := awsS3.NewFromConfig(cfg) - logger.Info("creating bucket if necessary") - _, err = client.CreateBucket(context.Background(), &awsS3.CreateBucketInput{ - Bucket: aws.String(config.S3Bucket), - }) - if err != nil && !strings.Contains(err.Error(), "BucketAlreadyOwnedByYou") { - return nil, fmt.Errorf("failed to create bucket: %w", err) - } - - logger.Info("s3 storage initialized") - - return &s3{config: config, client: client}, nil -} - -func (s *s3) Key(workspaceId string, dekID string) string { - return fmt.Sprintf("%s/%s", workspaceId, dekID) -} - -func (s *s3) Latest(workspaceId string) string { - return s.Key(workspaceId, "LATEST") -} - -func (s *s3) PutObject(ctx context.Context, key string, data []byte) error { - _, err := s.client.PutObject(ctx, &awsS3.PutObjectInput{ - Bucket: aws.String(s.config.S3Bucket), - Key: aws.String(key), - Body: bytes.NewReader(data), - }) - if err != nil { - return fmt.Errorf("failed to put object: %w", err) - } - return nil -} - -func (s *s3) GetObject(ctx context.Context, key string) ([]byte, bool, error) { - o, err := s.client.GetObject(ctx, &awsS3.GetObjectInput{ - Bucket: aws.String(s.config.S3Bucket), - Key: aws.String(key), - }) - if err != nil { - - if strings.Contains(err.Error(), "StatusCode: 404") { - return nil, false, nil - } - return nil, false, fmt.Errorf("failed to get object: %w", err) - } - defer func() { _ = o.Body.Close() }() - b, err := io.ReadAll(o.Body) - if err != nil { - return nil, false, fmt.Errorf("failed to read object: %w", err) - } - return b, true, nil -} - -func (s *s3) ListObjectKeys(ctx context.Context, prefix string) ([]string, error) { - input := &awsS3.ListObjectsV2Input{ - Bucket: aws.String(s.config.S3Bucket), - } - if prefix != "" { - input.Prefix = aws.String(prefix) - } - - o, err := s.client.ListObjectsV2(ctx, input) - if err != nil { - return nil, fmt.Errorf("failed to list objects: %w", err) - } - keys := make([]string, len(o.Contents)) - for i, obj := range o.Contents { - keys[i] = *obj.Key - } - return keys, nil -} diff --git a/svc/api/BUILD.bazel b/svc/api/BUILD.bazel index 88aa96c45a..74dcc85768 100644 --- a/svc/api/BUILD.bazel +++ b/svc/api/BUILD.bazel @@ -11,13 +11,13 @@ go_library( deps = [ "//gen/proto/cache/v1:cache", "//gen/proto/ctrl/v1/ctrlv1connect", + "//gen/proto/vault/v1/vaultv1connect", "//internal/services/analytics", "//internal/services/auditlogs", "//internal/services/caches", "//internal/services/keys", "//internal/services/ratelimit", "//internal/services/usagelimiter", - "//pkg/assert", "//pkg/clickhouse", "//pkg/clock", "//pkg/counter", @@ -31,7 +31,6 @@ go_library( "//pkg/runner", "//pkg/tls", "//pkg/vault", - "//pkg/vault/storage", "//pkg/version", "//pkg/zen", "//pkg/zen/validation", @@ -48,7 +47,6 @@ go_test( ":api", "//pkg/dockertest", "//pkg/uid", - "//pkg/vault/keys", "@com_github_stretchr_testify//require", ], ) diff --git a/svc/api/cancel_test.go b/svc/api/cancel_test.go index 395ecd3014..9c70c43e5e 100644 --- a/svc/api/cancel_test.go +++ b/svc/api/cancel_test.go @@ -11,7 +11,6 @@ import ( "github.com/stretchr/testify/require" "github.com/unkeyed/unkey/pkg/dockertest" "github.com/unkeyed/unkey/pkg/uid" - "github.com/unkeyed/unkey/pkg/vault/keys" "github.com/unkeyed/unkey/svc/api" ) @@ -30,9 +29,6 @@ func TestContextCancellation(t *testing.T) { // Create a cancellable context ctx, cancel := context.WithCancel(context.Background()) - _, masterKey, err := keys.GenerateMasterKey() - require.NoError(t, err) - // Configure the API server config := api.Config{ Platform: "test", @@ -46,7 +42,6 @@ func TestContextCancellation(t *testing.T) { DatabasePrimary: dbDsn, DatabaseReadonlyReplica: "", OtelEnabled: false, - VaultMasterKeys: []string{masterKey}, } // Create a channel to receive the result of the Run function diff --git a/svc/api/config.go b/svc/api/config.go index c779e818e9..1bae2ac831 100644 --- a/svc/api/config.go +++ b/svc/api/config.go @@ -4,7 +4,6 @@ import ( "net" "time" - "github.com/unkeyed/unkey/pkg/assert" "github.com/unkeyed/unkey/pkg/clock" "github.com/unkeyed/unkey/pkg/tls" ) @@ -14,13 +13,6 @@ const ( DefaultCacheInvalidationTopic = "cache-invalidations" ) -type S3Config struct { - URL string - Bucket string - AccessKeyID string - AccessKeySecret string -} - type Config struct { // InstanceID is the unique identifier for this instance of the API server InstanceID string @@ -82,8 +74,8 @@ type Config struct { TLSConfig *tls.Config // Vault Configuration - VaultMasterKeys []string - VaultS3 *S3Config + VaultURL string + VaultToken string // --- Kafka configuration --- @@ -137,17 +129,5 @@ type Config struct { func (c Config) Validate() error { // TLS configuration is validated when it's created from files // Other validations may be added here in the future - if c.VaultS3 != nil { - err := assert.All( - assert.NotEmpty(c.VaultS3.URL, "vault s3 url is empty"), - assert.NotEmpty(c.VaultS3.Bucket, "vault s3 bucket is empty"), - assert.NotEmpty(c.VaultS3.AccessKeyID, "vault s3 access key id is empty"), - assert.NotEmpty(c.VaultS3.AccessKeySecret, "vault s3 secret access key is empty"), - ) - if err != nil { - return err - } - } - return nil } diff --git a/svc/api/integration/harness.go b/svc/api/integration/harness.go index c27a69b8ad..0e8eac4c6b 100644 --- a/svc/api/integration/harness.go +++ b/svc/api/integration/harness.go @@ -137,6 +137,7 @@ func (h *Harness) RunAPI(config ApiConfig) *ApiCluster { mysqlHostCfg.DBName = "unkey" // Set the database name clickhouseHostDSN := containers.ClickHouse(h.t) kafkaBrokers := containers.Kafka(h.t) + vaultURL, vaultToken := containers.Vault(h.t) apiConfig := api.Config{ CacheInvalidationTopic: "", MaxRequestBodySize: 0, @@ -158,8 +159,8 @@ func (h *Harness) RunAPI(config ApiConfig) *ApiCluster { OtelTraceSamplingRate: 0.0, PrometheusPort: 0, TLSConfig: nil, - VaultMasterKeys: []string{"Ch9rZWtfMmdqMFBJdVhac1NSa0ZhNE5mOWlLSnBHenFPENTt7an5MRogENt9Si6wms4pQ2XIvqNSIgNpaBenJmXgcInhu6Nfv2U="}, // Test key from docker-compose - VaultS3: nil, + VaultURL: vaultURL, + VaultToken: vaultToken, KafkaBrokers: kafkaBrokers, // Use host brokers for test runner connections PprofEnabled: true, PprofUsername: "unkey", diff --git a/svc/api/internal/testutil/BUILD.bazel b/svc/api/internal/testutil/BUILD.bazel index a00af6ca6f..c5f928d941 100644 --- a/svc/api/internal/testutil/BUILD.bazel +++ b/svc/api/internal/testutil/BUILD.bazel @@ -28,12 +28,11 @@ go_library( "//pkg/testutil/containers", "//pkg/uid", "//pkg/vault", - "//pkg/vault/keys", - "//pkg/vault/storage", "//pkg/zen", "//pkg/zen/validation", "//svc/api/internal/middleware", "//svc/api/internal/testutil/seed", + "//svc/vault/testutil", "@com_connectrpc_connect//:connect", "@com_github_stretchr_testify//require", ], diff --git a/svc/api/internal/testutil/http.go b/svc/api/internal/testutil/http.go index 406ed440d0..c17126bdb8 100644 --- a/svc/api/internal/testutil/http.go +++ b/svc/api/internal/testutil/http.go @@ -27,12 +27,11 @@ import ( "github.com/unkeyed/unkey/pkg/testutil/containers" "github.com/unkeyed/unkey/pkg/uid" "github.com/unkeyed/unkey/pkg/vault" - masterKeys "github.com/unkeyed/unkey/pkg/vault/keys" - "github.com/unkeyed/unkey/pkg/vault/storage" "github.com/unkeyed/unkey/pkg/zen" "github.com/unkeyed/unkey/pkg/zen/validation" "github.com/unkeyed/unkey/svc/api/internal/middleware" "github.com/unkeyed/unkey/svc/api/internal/testutil/seed" + vaulttestutil "github.com/unkeyed/unkey/svc/vault/testutil" ) // Harness provides a complete integration test environment with real dependencies. @@ -62,7 +61,7 @@ type Harness struct { Auditlogs auditlogs.AuditLogService ClickHouse clickhouse.ClickHouse Ratelimit ratelimit.Service - Vault *vault.Service + Vault vault.Client AnalyticsConnectionManager analytics.ConnectionManager seeder *seed.Seeder } @@ -148,23 +147,8 @@ func NewHarness(t *testing.T) *Harness { }) require.NoError(t, err) - s3 := containers.S3(t) - - vaultStorage, err := storage.NewS3(storage.S3Config{ - S3URL: s3.HostURL, - S3Bucket: "test", - S3AccessKeyID: s3.AccessKeyID, - S3AccessKeySecret: s3.AccessKeySecret, - }) - require.NoError(t, err) - - _, masterKey, err := masterKeys.GenerateMasterKey() - require.NoError(t, err) - v, err := vault.New(vault.Config{ - Storage: vaultStorage, - MasterKeys: []string{masterKey}, - }) - require.NoError(t, err) + testVault := vaulttestutil.StartTestVaultWithMemory(t) + v := vault.NewConnectClient(testVault.Client) // Create analytics connection manager analyticsConnManager, err := analytics.NewConnectionManager(analytics.ConnectionManagerConfig{ diff --git a/svc/api/internal/testutil/seed/seed.go b/svc/api/internal/testutil/seed/seed.go index 3295ce71a8..fb05dea802 100644 --- a/svc/api/internal/testutil/seed/seed.go +++ b/svc/api/internal/testutil/seed/seed.go @@ -34,13 +34,13 @@ type Resources struct { type Seeder struct { t *testing.T DB db.Database - Vault *vault.Service + Vault vault.Client Resources Resources } // New creates a Seeder with the given database and vault service. Call [Seeder.Seed] // after creation to populate baseline data. -func New(t *testing.T, database db.Database, vault *vault.Service) *Seeder { +func New(t *testing.T, database db.Database, vault vault.Client) *Seeder { return &Seeder{ t: t, DB: database, diff --git a/svc/api/routes/services.go b/svc/api/routes/services.go index 92d0486b40..c8b03350bc 100644 --- a/svc/api/routes/services.go +++ b/svc/api/routes/services.go @@ -47,7 +47,7 @@ type Services struct { Caches caches.Caches // Vault provides encrypted storage for sensitive key material. - Vault *vault.Service + Vault vault.Client // ChproxyToken authenticates requests to internal chproxy endpoints. // When empty, chproxy routes are not registered. diff --git a/svc/api/routes/v2_apis_list_keys/handler.go b/svc/api/routes/v2_apis_list_keys/handler.go index afac577e30..8f80104d6a 100644 --- a/svc/api/routes/v2_apis_list_keys/handler.go +++ b/svc/api/routes/v2_apis_list_keys/handler.go @@ -30,7 +30,7 @@ type ( type Handler struct { DB db.Database Keys keys.KeyService - Vault *vault.Service + Vault vault.Client ApiCache cache.Cache[cache.ScopedKey, db.FindLiveApiByIDRow] } diff --git a/svc/api/routes/v2_keys_create_key/handler.go b/svc/api/routes/v2_keys_create_key/handler.go index 8bc1e82cd2..0f861edcf6 100644 --- a/svc/api/routes/v2_keys_create_key/handler.go +++ b/svc/api/routes/v2_keys_create_key/handler.go @@ -35,7 +35,7 @@ type Handler struct { DB db.Database Keys keys.KeyService Auditlogs auditlogs.AuditLogService - Vault *vault.Service + Vault vault.Client } // Method returns the HTTP method this route responds to diff --git a/svc/api/routes/v2_keys_get_key/handler.go b/svc/api/routes/v2_keys_get_key/handler.go index dec346b88f..2967b236be 100644 --- a/svc/api/routes/v2_keys_get_key/handler.go +++ b/svc/api/routes/v2_keys_get_key/handler.go @@ -29,7 +29,7 @@ type Handler struct { DB db.Database Keys keys.KeyService Auditlogs auditlogs.AuditLogService - Vault *vault.Service + Vault vault.Client } func (h *Handler) Method() string { diff --git a/svc/api/routes/v2_keys_reroll_key/handler.go b/svc/api/routes/v2_keys_reroll_key/handler.go index 543a4db166..7c7013c414 100644 --- a/svc/api/routes/v2_keys_reroll_key/handler.go +++ b/svc/api/routes/v2_keys_reroll_key/handler.go @@ -32,7 +32,7 @@ type Handler struct { DB db.Database Keys keys.KeyService Auditlogs auditlogs.AuditLogService - Vault *vault.Service + Vault vault.Client } // Method returns the HTTP method this route responds to diff --git a/svc/api/routes/v2_keys_whoami/handler.go b/svc/api/routes/v2_keys_whoami/handler.go index 8d2809a960..244f9bebc2 100644 --- a/svc/api/routes/v2_keys_whoami/handler.go +++ b/svc/api/routes/v2_keys_whoami/handler.go @@ -29,7 +29,7 @@ type Handler struct { DB db.Database Keys keys.KeyService Auditlogs auditlogs.AuditLogService - Vault *vault.Service + Vault vault.Client } func (h *Handler) Method() string { diff --git a/svc/api/run.go b/svc/api/run.go index 1a14d0b473..6e1499f916 100644 --- a/svc/api/run.go +++ b/svc/api/run.go @@ -12,6 +12,7 @@ import ( "connectrpc.com/connect" cachev1 "github.com/unkeyed/unkey/gen/proto/cache/v1" "github.com/unkeyed/unkey/gen/proto/ctrl/v1/ctrlv1connect" + "github.com/unkeyed/unkey/gen/proto/vault/v1/vaultv1connect" "github.com/unkeyed/unkey/internal/services/analytics" "github.com/unkeyed/unkey/internal/services/auditlogs" "github.com/unkeyed/unkey/internal/services/caches" @@ -30,7 +31,6 @@ import ( "github.com/unkeyed/unkey/pkg/rpc/interceptor" "github.com/unkeyed/unkey/pkg/runner" "github.com/unkeyed/unkey/pkg/vault" - "github.com/unkeyed/unkey/pkg/vault/storage" "github.com/unkeyed/unkey/pkg/version" "github.com/unkeyed/unkey/pkg/zen" "github.com/unkeyed/unkey/pkg/zen/validation" @@ -175,26 +175,16 @@ func Run(ctx context.Context, cfg Config) error { return fmt.Errorf("unable to create usage limiter service: %w", err) } - var vaultSvc *vault.Service - if len(cfg.VaultMasterKeys) > 0 && cfg.VaultS3 != nil { - var vaultStorage storage.Storage - vaultStorage, err = storage.NewS3(storage.S3Config{ - S3URL: cfg.VaultS3.URL, - S3Bucket: cfg.VaultS3.Bucket, - S3AccessKeyID: cfg.VaultS3.AccessKeyID, - S3AccessKeySecret: cfg.VaultS3.AccessKeySecret, - }) - if err != nil { - return fmt.Errorf("unable to create vault storage: %w", err) - } - - vaultSvc, err = vault.New(vault.Config{ - Storage: vaultStorage, - MasterKeys: cfg.VaultMasterKeys, - }) - if err != nil { - return fmt.Errorf("unable to create vault service: %w", err) - } + var vaultClient vault.Client + if cfg.VaultURL != "" { + connectClient := vaultv1connect.NewVaultServiceClient( + &http.Client{}, + cfg.VaultURL, + connect.WithInterceptors(interceptor.NewHeaderInjector(map[string]string{ + "Authorization": fmt.Sprintf("Bearer %s", cfg.VaultToken), + })), + ) + vaultClient = vault.NewConnectClient(connectClient) } auditlogSvc, err := auditlogs.New(auditlogs.Config{ @@ -254,13 +244,13 @@ func Run(ctx context.Context, cfg Config) error { // Initialize analytics connection manager analyticsConnMgr := analytics.NewNoopConnectionManager() - if cfg.ClickhouseAnalyticsURL != "" && vaultSvc != nil { + if cfg.ClickhouseAnalyticsURL != "" && vaultClient != nil { analyticsConnMgr, err = analytics.NewConnectionManager(analytics.ConnectionManagerConfig{ SettingsCache: caches.ClickhouseSetting, Database: db, Clock: clk, BaseURL: cfg.ClickhouseAnalyticsURL, - Vault: vaultSvc, + Vault: vaultClient, }) if err != nil { return fmt.Errorf("unable to create analytics connection manager: %w", err) @@ -286,7 +276,7 @@ func Run(ctx context.Context, cfg Config) error { Ratelimit: rlSvc, Auditlogs: auditlogSvc, Caches: caches, - Vault: vaultSvc, + Vault: vaultClient, ChproxyToken: cfg.ChproxyToken, CtrlDeploymentClient: ctrlDeploymentClient, PprofEnabled: cfg.PprofEnabled, diff --git a/web/apps/engineering/content/docs/cli/run/api/index.mdx b/web/apps/engineering/content/docs/cli/run/api/index.mdx index 30e25b64c3..3e354ac9f7 100644 --- a/web/apps/engineering/content/docs/cli/run/api/index.mdx +++ b/web/apps/engineering/content/docs/cli/run/api/index.mdx @@ -134,39 +134,18 @@ Path to TLS key file for HTTPS. Both cert and key must be provided to enable HTT - **Environment:** `UNKEY_TLS_KEY_FILE` - -Vault master keys for encryption - -- **Type:** string[] -- **Environment:** `UNKEY_VAULT_MASTER_KEYS` - - - -S3 Compatible Endpoint URL - -- **Type:** string -- **Environment:** `UNKEY_VAULT_S3_URL` - - - -S3 bucket name - -- **Type:** string -- **Environment:** `UNKEY_VAULT_S3_BUCKET` - - - -S3 access key ID + +URL of the remote vault service for encryption/decryption - **Type:** string -- **Environment:** `UNKEY_VAULT_S3_ACCESS_KEY_ID` +- **Environment:** `UNKEY_VAULT_URL` - -S3 secret access key + +Bearer token for vault service authentication - **Type:** string -- **Environment:** `UNKEY_VAULT_S3_ACCESS_KEY_SECRET` +- **Environment:** `UNKEY_VAULT_TOKEN`