chore: use vault in api

.github/workflows/job_bazel.yaml

@@ -33,6 +33,6 @@ jobs:
       # Running containers is temporary until we moved them inside of bazel,
       # at that point they are only created if they are actually needed
       - name: Start containers
-        run: docker compose -f ./dev/docker-compose.yaml up s3 clickhouse kafka mysql -d --wait
+        run: docker compose -f ./dev/docker-compose.yaml up s3 clickhouse kafka mysql vault -d --wait
       - name: Run tests
         run: bazel test //... --test_output=errors

Makefile

@@ -91,7 +91,7 @@ generate: generate-sql ## Generate code from protobuf and other sources
 
 .PHONY: test
 test: ## Run tests with bazel
-	docker compose -f ./dev/docker-compose.yaml up -d mysql clickhouse s3 kafka --wait
+	docker compose -f ./dev/docker-compose.yaml up -d mysql clickhouse s3 kafka vault --wait
 	bazel test //...
 	make clean-docker-test
 

cmd/api/main.go

@@ -68,16 +68,10 @@ var Cmd = &cli.Command{
 			cli.EnvVar("UNKEY_TLS_KEY_FILE")),
 
 		// Vault Configuration
-		cli.StringSlice("vault-master-keys", "Vault master keys for encryption",
-			cli.EnvVar("UNKEY_VAULT_MASTER_KEYS")),
-		cli.String("vault-s3-url", "S3 Compatible Endpoint URL",
-			cli.EnvVar("UNKEY_VAULT_S3_URL")),
-		cli.String("vault-s3-bucket", "S3 bucket name",
-			cli.EnvVar("UNKEY_VAULT_S3_BUCKET")),
-		cli.String("vault-s3-access-key-id", "S3 access key ID",
-			cli.EnvVar("UNKEY_VAULT_S3_ACCESS_KEY_ID")),
-		cli.String("vault-s3-access-key-secret", "S3 secret access key",
-			cli.EnvVar("UNKEY_VAULT_S3_ACCESS_KEY_SECRET")),
+		cli.String("vault-url", "URL of the remote vault service for encryption/decryption",
+			cli.EnvVar("UNKEY_VAULT_URL")),
+		cli.String("vault-token", "Bearer token for vault service authentication",
+			cli.EnvVar("UNKEY_VAULT_TOKEN")),
 
 		// Kafka Configuration
 		cli.StringSlice("kafka-brokers", "Comma-separated list of Kafka broker addresses for distributed cache invalidation",
@@ -146,16 +140,6 @@ func action(ctx context.Context, cmd *cli.Command) error {
 		}
 	}
 
-	var vaultS3Config *api.S3Config
-	if cmd.String("vault-s3-url") != "" {
-		vaultS3Config = &api.S3Config{
-			URL:             cmd.String("vault-s3-url"),
-			Bucket:          cmd.String("vault-s3-bucket"),
-			AccessKeyID:     cmd.String("vault-s3-access-key-id"),
-			AccessKeySecret: cmd.String("vault-s3-access-key-secret"),
-		}
-	}
-
 	config := api.Config{
 		// Basic configuration
 		CacheInvalidationTopic: "",
@@ -189,8 +173,8 @@ func action(ctx context.Context, cmd *cli.Command) error {
 		Listener: nil, // Production uses HttpPort
 
 		// Vault configuration
-		VaultMasterKeys: cmd.StringSlice("vault-master-keys"),
-		VaultS3:         vaultS3Config,
+		VaultURL:   cmd.String("vault-url"),
+		VaultToken: cmd.String("vault-token"),
 
 		// Kafka configuration
 		KafkaBrokers: cmd.StringSlice("kafka-brokers"),

dev/docker-compose.yaml

@@ -70,7 +70,7 @@ services:
     depends_on:
       mysql:
         condition: service_healthy
-      s3:
+      vault:
         condition: service_healthy
       redis:
         condition: service_healthy
@@ -87,11 +87,8 @@ services:
       UNKEY_CLICKHOUSE_URL: "clickhouse://default:password@clickhouse:9000?secure=false&skip_verify=true"
       UNKEY_CHPROXY_AUTH_TOKEN: "chproxy-test-token-123"
       UNKEY_OTEL: false
-      UNKEY_VAULT_S3_URL: "http://s3:3902"
-      UNKEY_VAULT_S3_BUCKET: "vault"
-      UNKEY_VAULT_S3_ACCESS_KEY_ID: "minio_root_user"
-      UNKEY_VAULT_S3_ACCESS_KEY_SECRET: "minio_root_password"
-      UNKEY_VAULT_MASTER_KEYS: "Ch9rZWtfMmdqMFBJdVhac1NSa0ZhNE5mOWlLSnBHenFPENTt7an5MRogENt9Si6wms4pQ2XIvqNSIgNpaBenJmXgcInhu6Nfv2U="
+      UNKEY_VAULT_URL: "http://vault:8060"
+      UNKEY_VAULT_TOKEN: "vault-test-token-123"
       UNKEY_KAFKA_BROKERS: "kafka:9092"
       UNKEY_CLICKHOUSE_ANALYTICS_URL: "http://clickhouse:8123/default"
       UNKEY_CTRL_URL: "http://ctrl-api:7091"

dev/k8s/manifests/api.yaml

@@ -56,16 +56,10 @@ spec:
             - name: UNKEY_PROMETHEUS_PORT
               value: "0"
             # Vault Configuration
-            - name: UNKEY_VAULT_MASTER_KEYS
-              value: "Ch9rZWtfMmdqMFBJdVhac1NSa0ZhNE5mOWlLSnBHenFPENTt7an5MRogENt9Si6wms4pQ2XIvqNSIgNpaBenJmXgcInhu6Nfv2U="
-            - name: UNKEY_VAULT_S3_URL
-              value: "http://s3:3902"
-            - name: UNKEY_VAULT_S3_BUCKET
-              value: "vault"
-            - name: UNKEY_VAULT_S3_ACCESS_KEY_ID
-              value: "minio_root_user"
-            - name: UNKEY_VAULT_S3_ACCESS_KEY_SECRET
-              value: "minio_root_password"
+            - name: UNKEY_VAULT_URL
+              value: "http://vault:8060"
+            - name: UNKEY_VAULT_TOKEN
+              value: "vault-test-token-123"
             # ClickHouse Proxy Service Configuration
             - name: UNKEY_CHPROXY_AUTH_TOKEN
               value: "chproxy-test-token-123"

internal/services/analytics/service.go

@@ -23,7 +23,7 @@ type connectionManager struct {
 	connectionCache cache.Cache[string, clickhouse.ClickHouse]
 	database        db.Database
 	baseURL         string
-	vault           *vault.Service
+	vault           vault.Client
 }
 
 // ConnectionManagerConfig contains configuration for the connection manager
@@ -32,7 +32,7 @@ type ConnectionManagerConfig struct {
 	Database      db.Database
 	Clock         clock.Clock
 	BaseURL       string // e.g., "http://clickhouse:8123/default" or "clickhouse://clickhouse:9000/default"
-	Vault         *vault.Service
+	Vault         vault.Client
 }
 
 // NewConnectionManager creates a new connection manager

pkg/testutil/containers/containers.go

@@ -178,6 +178,19 @@ func OTEL(t *testing.T) OTELConfig {
 	}
 }
 
+// Vault returns the URL and bearer token for the vault service in integration testing.
+//
+// The vault service runs on port 8060 and requires a bearer token for authentication.
+// These values match the vault service configuration in docker-compose.yaml.
+//
+// Example usage:
+//
+//	vaultURL, vaultToken := containers.Vault(t)
+//	client := vaultv1connect.NewVaultServiceClient(httpClient, vaultURL, ...)
+func Vault(t *testing.T) (string, string) {
+	return "http://localhost:8060", "vault-test-token-123"
+}
+
 // Kafka returns Kafka broker addresses for integration testing.
 //
 // Returns broker addresses for connecting to the Kafka service running

pkg/vault/BUILD.bazel

@@ -3,26 +3,14 @@ load("@rules_go//go:def.bzl", "go_library")
 go_library(
     name = "vault",
     srcs = [
-        "create_dek.go",
-        "decrypt.go",
-        "encrypt.go",
-        "reencrypt.go",
-        "roll_deks.go",
-        "service.go",
+        "client.go",
+        "connect_client.go",
     ],
     importpath = "github.com/unkeyed/unkey/pkg/vault",
     visibility = ["//visibility:public"],
     deps = [
         "//gen/proto/vault/v1:vault",
-        "//pkg/cache",
-        "//pkg/cache/middleware",
-        "//pkg/clock",
-        "//pkg/encryption",
-        "//pkg/logger",
-        "//pkg/otel/tracing",
-        "//pkg/vault/keyring",
-        "//pkg/vault/storage",
-        "@io_opentelemetry_go_otel//attribute",
-        "@org_golang_google_protobuf//proto",
+        "//gen/proto/vault/v1/vaultv1connect",
+        "@com_connectrpc_connect//:connect",
     ],
 )

pkg/vault/client.go

@@ -0,0 +1,14 @@
+package vault
+
+import (
+	"context"
+
+	vaultv1 "github.com/unkeyed/unkey/gen/proto/vault/v1"
+)
+
+// Client defines the interface for vault encryption and decryption operations.
+// [ConnectClient] implements this interface by wrapping a remote vault service.
+type Client interface {
+	Encrypt(ctx context.Context, req *vaultv1.EncryptRequest) (*vaultv1.EncryptResponse, error)
+	Decrypt(ctx context.Context, req *vaultv1.DecryptRequest) (*vaultv1.DecryptResponse, error)
+}

pkg/vault/connect_client.go

@@ -0,0 +1,39 @@
+package vault
+
+import (
+	"context"
+
+	"connectrpc.com/connect"
+	vaultv1 "github.com/unkeyed/unkey/gen/proto/vault/v1"
+	"github.com/unkeyed/unkey/gen/proto/vault/v1/vaultv1connect"
+)
+
+// Compile-time check that *ConnectClient implements Client.
+var _ Client = (*ConnectClient)(nil)
+
+// ConnectClient adapts a [vaultv1connect.VaultServiceClient] to the [Client] interface,
+// wrapping and unwrapping connect.Request/Response types.
+type ConnectClient struct {
+	inner vaultv1connect.VaultServiceClient
+}
+
+// NewConnectClient creates a new [ConnectClient] wrapping the given connect client.
+func NewConnectClient(inner vaultv1connect.VaultServiceClient) *ConnectClient {
+	return &ConnectClient{inner: inner}
+}
+
+func (c *ConnectClient) Encrypt(ctx context.Context, req *vaultv1.EncryptRequest) (*vaultv1.EncryptResponse, error) {
+	resp, err := c.inner.Encrypt(ctx, connect.NewRequest(req))
+	if err != nil {
+		return nil, err
+	}
+	return resp.Msg, nil
+}
+
+func (c *ConnectClient) Decrypt(ctx context.Context, req *vaultv1.DecryptRequest) (*vaultv1.DecryptResponse, error) {
+	resp, err := c.inner.Decrypt(ctx, connect.NewRequest(req))
+	if err != nil {
+		return nil, err
+	}
+	return resp.Msg, nil
+}