From 2ba59219e6773c55272da18e8d9c20e9f5b8b001 Mon Sep 17 00:00:00 2001 From: Flo Date: Wed, 10 Sep 2025 21:26:31 +0200 Subject: [PATCH 1/9] chore(setup): k8s for local dev-exp --- go/Dockerfile | 7 +- go/Dockerfile.tilt | 9 + go/K8S_DEVELOPMENT.md | 258 ++++++++++++++++++++++++ go/Makefile | 108 +++++++++- go/Tiltfile | 302 ++++++++++++++++++++++++++++ go/k8s/manifests/api.yaml | 113 +++++++++++ go/k8s/manifests/clickhouse.yaml | 94 +++++++++ go/k8s/manifests/ctrl.yaml | 104 ++++++++++ go/k8s/manifests/dashboard.yaml | 87 ++++++++ go/k8s/manifests/gw.yaml | 122 +++++++++++ go/k8s/manifests/ingress.yaml | 2 + go/k8s/manifests/mysql.yaml | 90 +++++++++ go/k8s/manifests/namespace.yaml | 7 + go/k8s/manifests/observability.yaml | 197 ++++++++++++++++++ go/k8s/manifests/planetscale.yaml | 67 ++++++ go/k8s/manifests/rbac.yaml | 42 ++++ go/k8s/manifests/redis.yaml | 61 ++++++ go/k8s/manifests/s3.yaml | 113 +++++++++++ 18 files changed, 1781 insertions(+), 2 deletions(-) create mode 100644 go/Dockerfile.tilt create mode 100644 go/K8S_DEVELOPMENT.md create mode 100644 go/Tiltfile create mode 100644 go/k8s/manifests/api.yaml create mode 100644 go/k8s/manifests/clickhouse.yaml create mode 100644 go/k8s/manifests/ctrl.yaml create mode 100644 go/k8s/manifests/dashboard.yaml create mode 100644 go/k8s/manifests/gw.yaml create mode 100644 go/k8s/manifests/ingress.yaml create mode 100644 go/k8s/manifests/mysql.yaml create mode 100644 go/k8s/manifests/namespace.yaml create mode 100644 go/k8s/manifests/observability.yaml create mode 100644 go/k8s/manifests/planetscale.yaml create mode 100644 go/k8s/manifests/rbac.yaml create mode 100644 go/k8s/manifests/redis.yaml create mode 100644 go/k8s/manifests/s3.yaml diff --git a/go/Dockerfile b/go/Dockerfile index a11a67fa68..5fa975b02b 100644 --- a/go/Dockerfile +++ b/go/Dockerfile @@ -1,3 +1,5 @@ +ARG BASE_IMAGE=gcr.io/distroless/static-debian12:latest + FROM golang:1.24 AS builder WORKDIR /go/src/github.com/unkeyed/unkey/go @@ -9,7 +11,10 @@ ARG VERSION ENV CGO_ENABLED=0 RUN go build -o bin/unkey -ldflags="-X 'github.com/unkeyed/unkey/go/pkg/version.Version=${VERSION}'" ./main.go -FROM gcr.io/distroless/static-debian12 +FROM ${BASE_IMAGE} + +# When BASE_IMAGE is golang-alpine (for Tilt dev), Go tools are already available + COPY --from=builder /go/src/github.com/unkeyed/unkey/go/bin/unkey / LABEL org.opencontainers.image.source=https://github.com/unkeyed/unkey/go diff --git a/go/Dockerfile.tilt b/go/Dockerfile.tilt new file mode 100644 index 0000000000..f6d07e1421 --- /dev/null +++ b/go/Dockerfile.tilt @@ -0,0 +1,9 @@ +FROM busybox:latest + +# Copy the pre-built binary +COPY bin/unkey /unkey + +LABEL org.opencontainers.image.source=https://github.com/unkeyed/unkey/go +LABEL org.opencontainers.image.description="Unkey API" + +ENTRYPOINT ["/unkey"] \ No newline at end of file diff --git a/go/K8S_DEVELOPMENT.md b/go/K8S_DEVELOPMENT.md new file mode 100644 index 0000000000..9e5e7daff4 --- /dev/null +++ b/go/K8S_DEVELOPMENT.md @@ -0,0 +1,258 @@ +# Kubernetes Development Setup + +This guide shows how to run Unkey services locally using Docker Desktop or OrbStack Kubernetes instead of Docker Compose. + +## Prerequisites + +Install the required tools: + +- **Docker Desktop with Kubernetes enabled** OR **OrbStack with Kubernetes enabled** +- **kubectl**: https://kubernetes.io/docs/tasks/tools/ + +Or check if already installed: +```bash +make k8s-check +``` + +Optional for enhanced development experience: +- **Tilt**: https://docs.tilt.dev/install.html + +## Quick Start + +### Option 1: Full Environment (Recommended) +```bash +make k8s-up +``` + +This will: +- Use your current Kubernetes cluster (Docker Desktop/OrbStack) +- Build and deploy all services (MySQL, ClickHouse, S3, Observability, Unkey) +- Wait for all services to be ready +- Show connection info + +### Option 2: Individual Services +```bash +# Start only MySQL +make start-mysql + +# Start only ctrl (requires cluster to exist) +make start-ctrl + +# Start all services individually +make start-all +``` + +### Option 3: Enhanced Development with Tilt +```bash +make dev +``` + +If Tilt is installed, this provides: +- Hot reloading for Go code changes +- Unified log viewing in web UI (http://localhost:10350) +- Resource management dashboard +- Automatic rebuilds on file changes +- Selective service startup + +## Accessing Services + +### Via NodePort (OrbStack) + +Services are accessible directly on localhost via NodePort. Check actual port assignments with: + +```bash +make k8s-ports +``` + +This will show the randomly assigned NodePorts, e.g.: +- **Dashboard**: http://localhost:32001 +- **API**: http://localhost:32002 +- **Gateway**: http://localhost:32003 +- **Ctrl**: http://localhost:32004 +- **Prometheus**: http://localhost:32005 +- **S3 Console**: http://localhost:32006 + +### Alternative: LoadBalancer domains + +OrbStack also supports LoadBalancer services with automatic `*.k8s.orb.local` domains, but NodePort is simpler for development. + +### Inside the cluster +- **MySQL**: `mysql.unkey.svc.cluster.local:3306` +- **ClickHouse**: `clickhouse.unkey.svc.cluster.local:8123` +- **S3**: `s3.unkey.svc.cluster.local:9000` +- **Prometheus**: `prometheus.unkey.svc.cluster.local:9090` +- **OTEL Collector**: `otel-collector.unkey.svc.cluster.local:4317` + +## Development Workflow + +### Making Code Changes + +#### With Tilt (Hot Reloading) +1. Start Tilt: `make dev` +2. Edit Go files +3. Changes automatically rebuild and restart services +4. View logs in Tilt UI + +#### Without Tilt (Manual) +1. Make code changes +2. Rebuild and redeploy: + ```bash + make start-ctrl # Rebuilds and redeploys ctrl + ``` + +### Managing the Environment + +```bash +# Check status +make k8s-status + +# Reset everything (delete and recreate) +make k8s-reset + +# Stop everything +make k8s-down +``` + +### Debugging + +```bash +# View logs +kubectl logs -n unkey -l app=ctrl -f +kubectl logs -n unkey -l app=mysql -f + +# Get shell access +kubectl exec -n unkey -it deployment/ctrl -- /bin/sh +kubectl exec -n unkey -it deployment/mysql -- /bin/bash + +# Check service health +kubectl get pods -n unkey +kubectl describe pod -n unkey +``` + +## Configuration + +### Environment Variables +The ctrl service uses these key environment variables: +- `UNKEY_DATABASE_PRIMARY`: Connection to partition database +- `UNKEY_DATABASE_HYDRA`: Connection to hydra database +- `UNKEY_HTTP_PORT`: Service port (8084) +- `UNKEY_PLATFORM`: Set to "kubernetes" + +### Database Setup +MySQL is automatically configured with: +- `unkey` database for main data +- `hydra` database for OAuth flows +- `partition_001` database for partitioned data +- User `unkey` with password `password` + +All schemas are automatically applied on startup. + +## Customization + +### Selective Services with Tilt +```bash +# Start only database services +tilt up mysql,clickhouse,planetscale + +# Start only storage services +tilt up s3 + +# Start only observability +tilt up observability + +# Start only Unkey services +tilt up ctrl +``` + +### Custom Resource Limits +Edit `k8s/manifests/ctrl.yaml` to adjust: +```yaml +resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "512Mi" + cpu: "500m" +``` + +### Persistent Data +MySQL data persists between restarts via PersistentVolumeClaim. To reset: +```bash +kubectl delete pvc -n unkey mysql-pvc +make start-mysql +``` + +## Troubleshooting + +### Common Issues + +#### Cluster Not Available +```bash +# Make sure Kubernetes is enabled in Docker Desktop/OrbStack +# Check current context +kubectl config current-context + +# Switch context if needed +kubectl config use-context docker-desktop # or orbstack +``` + +#### Services Not Ready +```bash +# Check events +kubectl get events -n unkey --sort-by='.lastTimestamp' + +# Check pod logs +kubectl logs -n unkey -l app=mysql +kubectl logs -n unkey -l app=ctrl +``` + +#### Port Conflicts +If ports 8084 or 3306 are in use: +```bash +# Check what's using the port +lsof -i :8084 +lsof -i :3306 + +# Kill other services or change ports in k8s manifests +``` + +#### Image Build Issues +```bash +# Manual build +docker build -t unkey/mysql:latest -f ../deployment/Dockerfile.mysql ../ +docker build -t unkey:latest . +``` + +### Performance Tips +- Use `make dev` for fastest development cycle +- Keep cluster running between sessions (don't run `k8s-down`) +- Use selective service startup when working on specific components + +## vs Docker Compose + +| Feature | Kubernetes | Docker Compose | +|---------|-----------|----------------| +| **Prod Similarity** | ✅ Real Kubernetes | ❌ Different from prod | +| **Resource Usage** | ✅ Native containers | ✅ Native containers | +| **Setup Complexity** | ⚠️ Enable K8s in Docker/OrbStack | ✅ Simple setup | +| **Service Discovery** | ✅ K8s native DNS | ⚠️ Docker networks | +| **Scaling** | ✅ Easy horizontal scaling | ❌ Limited scaling | +| **Hot Reloading** | ✅ With Tilt | ⚠️ Manual restarts | +| **Debugging** | ✅ Rich tooling | ✅ Simple logs | +| **Architecture** | ✅ Clean separation | ✅ Clean separation | + +Choose Kubernetes for true prod-like development, Docker Compose for quick iterations. + +## Available Services + +All services can be started individually or together: + +- `mysql` - MySQL database with schemas +- `clickhouse` - ClickHouse analytics database +- `s3` - MinIO object storage +- `planetscale` - PlanetScale HTTP database proxy +- `observability` - Prometheus + OTEL Collector +- `ctrl` - Main Unkey service (ctrl/api/gw) + +Use `make k8s-up` to start everything or selectively with Tilt. \ No newline at end of file diff --git a/go/Makefile b/go/Makefile index ad10d9ebc5..f5e20de3f1 100644 --- a/go/Makefile +++ b/go/Makefile @@ -1,4 +1,4 @@ -.PHONY: install tools fmt test-unit test-integration test-integration-long test-stress test build generate pull up clean +.PHONY: install tools fmt test-unit test-integration test-integration-long test-stress test build generate pull up clean k8s-check k8s-up k8s-down k8s-reset k8s-status start-mysql start-ctrl start-all dev # Detect OS and set GOMAXPROCS accordingly UNAME_S := $(shell uname -s) @@ -78,3 +78,109 @@ test-integration: export SIMULATION_TEST=false test-integration: up @echo "Running integration tests w/$(PARALLEL_PROCS) parallel test processes" @go test -tags=integration -json -failfast -timeout=15m -parallel=$(PARALLEL_PROCS) ./... | tparse -all -progress -smallscreen + +# ============================================================================ +# Kubernetes Development Commands +# ============================================================================ + +k8s-check: ## Check if kubectl is available and cluster is running + @command -v kubectl >/dev/null 2>&1 || { echo "ERROR: kubectl not found. Install from: https://kubernetes.io/docs/tasks/tools/"; exit 1; } + @kubectl cluster-info >/dev/null 2>&1 || { echo "ERROR: Kubernetes cluster not available. Enable Kubernetes in Docker Desktop/OrbStack"; exit 1; } + @echo "Kubernetes cluster is available" + +k8s-up: k8s-check ## Deploy all services to current Kubernetes cluster + @echo "Building Docker images..." + @docker build -t unkey/mysql:latest -f ../deployment/Dockerfile.mysql ../ + @docker build -t unkey:latest . + @echo "Creating namespace..." + @kubectl apply -f k8s/manifests/namespace.yaml + @echo "Applying Kubernetes manifests..." + @kubectl apply -f k8s/manifests/ + @echo "Waiting for services to be ready..." + @kubectl wait --for=condition=ready pod -l app=mysql -n unkey --timeout=180s + @kubectl wait --for=condition=ready pod -l app=clickhouse -n unkey --timeout=180s + @kubectl wait --for=condition=ready pod -l app=s3 -n unkey --timeout=180s + @kubectl wait --for=condition=ready pod -l app=api -n unkey --timeout=180s + @kubectl wait --for=condition=ready pod -l app=gw -n unkey --timeout=180s + @kubectl wait --for=condition=ready pod -l app=ctrl -n unkey --timeout=180s + @kubectl wait --for=condition=ready pod -l app=dashboard -n unkey --timeout=180s + @echo "Kubernetes environment is ready!" + @echo "" + @echo "Services accessible via NodePort on localhost - check actual ports with:" + @echo "" + @echo "Check NodePort assignments: make k8s-ports" + @make k8s-status + +k8s-down: ## Delete all services from current Kubernetes cluster + @echo "Deleting all services..." + @kubectl delete namespace unkey --ignore-not-found=true + @echo "Services deleted" + +k8s-reset: k8s-down k8s-up ## Reset the entire Kubernetes environment + +k8s-status: ## Show status of k8s services + @echo "Cluster Status:" + @kubectl get nodes -o wide 2>/dev/null || echo "ERROR: Cluster not running" + @echo "\nServices Status:" + @kubectl get pods,svc -n unkey 2>/dev/null || echo "ERROR: Services not deployed" + +k8s-ports: ## Show NodePort assignments for direct localhost access + @echo "Services accessible via NodePort on localhost:" + @kubectl get svc -n unkey -o custom-columns="NAME:.metadata.name,TYPE:.spec.type,PORT(S):.spec.ports[*].port,NODEPORT:.spec.ports[*].nodePort" | grep NodePort + +# Helper function to deploy a service +define deploy-service + @echo "Starting $(2)..." + $(if $(3),@docker build -t $(3) $(4)) + @kubectl apply -f k8s/manifests/namespace.yaml + @kubectl apply -f k8s/manifests/$(1).yaml + @kubectl wait --for=condition=ready pod -l app=$(1) -n unkey --timeout=180s + $(if $(5),@kubectl wait --for=condition=ready pod -l app=$(5) -n unkey --timeout=180s) + @echo "$(2) is ready!" +endef + +start-mysql: k8s-check ## Deploy only MySQL + $(call deploy-service,mysql,MySQL,unkey/mysql:latest,-f ../deployment/Dockerfile.mysql ../) + +start-clickhouse: k8s-check ## Deploy only ClickHouse + $(call deploy-service,clickhouse,ClickHouse) + +start-redis: k8s-check ## Deploy only Redis + $(call deploy-service,redis,Redis) + +start-s3: k8s-check ## Deploy only S3 (MinIO) + $(call deploy-service,s3,S3 (MinIO)) + +start-planetscale: k8s-check ## Deploy only PlanetScale HTTP driver + $(call deploy-service,planetscale,PlanetScale HTTP driver) + +start-observability: k8s-check ## Deploy only Observability stack + $(call deploy-service,observability,Observability stack,,,otel-collector) + +start-api: k8s-check ## Deploy only API service (3 replicas) + $(call deploy-service,api,API,unkey:latest,.) + +start-gw: k8s-check ## Deploy only Gateway service + $(call deploy-service,gw,Gateway,unkey:latest,.) + +start-ctrl: k8s-check ## Deploy only ctrl service + $(call deploy-service,ctrl,Control Plane,unkey:latest,.) + +start-dashboard: k8s-check ## Deploy only dashboard service + $(call deploy-service,dashboard,Dashboard,unkey/dashboard:latest,-f ../apps/dashboard/Dockerfile ../) + +start-unkey-services: start-api start-gw start-ctrl start-dashboard ## Deploy all Unkey services + +start-dependencies: start-mysql start-clickhouse start-redis start-s3 start-planetscale start-observability ## Deploy all dependency services + +start-all: start-dependencies start-unkey-services ## Deploy all services individually + +dev: ## Start with Tilt (if available) or fallback to k8s-up + @if command -v tilt >/dev/null 2>&1; then \ + echo "Starting with Tilt..."; \ + tilt up; \ + else \ + echo "Tilt not found, using k8s-up instead"; \ + echo "Install Tilt from: https://docs.tilt.dev/install.html"; \ + make k8s-up; \ + fi diff --git a/go/Tiltfile b/go/Tiltfile new file mode 100644 index 0000000000..5c0ca9df1f --- /dev/null +++ b/go/Tiltfile @@ -0,0 +1,302 @@ +# Tiltfile for Unkey development +# This provides an enhanced development experience with hot reloading and unified logs + +# Load Tilt extensions +load('ext://namespace', 'namespace_create') +load('ext://restart_process', 'docker_build_with_restart') + +# Tilt configuration +config.define_string_list("services", args=False, usage="Services to run (mysql,ctrl,clickhouse,s3,observability,planetscale or all)") +config.define_bool("debug", args=False, usage="Enable debug mode") + +cfg = config.parse() +services = cfg.get('services', ['all']) +debug_mode = cfg.get('debug', False) + +print("Tilt starting with services: %s" % services) + +# Create namespace using the extension with allow_duplicates +namespace_create('unkey', allow_duplicates=True) + +# Define which services to start +start_mysql = 'all' in services or 'mysql' in services +start_clickhouse = 'all' in services or 'clickhouse' in services +start_s3 = 'all' in services or 's3' in services +start_observability = 'all' in services or 'observability' in services +start_planetscale = 'all' in services or 'planetscale' in services +start_api = 'all' in services or 'api' in services +start_gw = 'all' in services or 'gateway' in services or 'gw' in services +start_ctrl = 'all' in services or 'ctrl' in services +start_dashboard = 'all' in services or 'dashboard' in services + +# Apply RBAC and ingress +k8s_yaml('k8s/manifests/rbac.yaml') +k8s_yaml('k8s/manifests/ingress.yaml') + +# Redis service +redis_started = False +if start_api: # Redis is needed by API service + print("Setting up Redis...") + k8s_yaml('k8s/manifests/redis.yaml') + k8s_resource( + 'redis', + port_forwards='6379:6379', + resource_deps=[], + labels=['database'] + ) + redis_started = True + +# MySQL service +if start_mysql: + print("Setting up MySQL...") + # Only build if image doesn't exist + if not local('docker images -q unkey/mysql:latest 2>/dev/null', quiet=True): + print("Building MySQL image...") + docker_build( + 'unkey/mysql:latest', + '../', # Build context at repo root + dockerfile='../deployment/Dockerfile.mysql' + ) + else: + print("Using existing MySQL image") + k8s_yaml('k8s/manifests/mysql.yaml') + k8s_resource( + 'mysql', + port_forwards='3306:3306', + resource_deps=[], + labels=['database'] + ) + +# ClickHouse service +if start_clickhouse: + print("Setting up ClickHouse...") + k8s_yaml('k8s/manifests/clickhouse.yaml') + k8s_resource( + 'clickhouse', + port_forwards='8123:8123', + resource_deps=[], + labels=['database'] + ) + +# S3 (MinIO) service +if start_s3: + print("Setting up S3 (MinIO)...") + k8s_yaml('k8s/manifests/s3.yaml') + k8s_resource( + 's3', + port_forwards=['9000:9000', '9001:9001'], + resource_deps=[], + labels=['storage'] + ) + +# PlanetScale HTTP driver +if start_planetscale: + print("Setting up PlanetScale HTTP driver...") + k8s_yaml('k8s/manifests/planetscale.yaml') + deps = ['mysql'] if start_mysql else [] + k8s_resource( + 'planetscale', + resource_deps=deps, + labels=['database'] + ) + +# Observability stack +if start_observability: + print("Setting up observability stack...") + k8s_yaml('k8s/manifests/observability.yaml') + k8s_resource( + 'prometheus', + port_forwards='9090:9090', + resource_deps=[], + labels=['observability'] + ) + k8s_resource( + 'otel-collector', + port_forwards=['4317:4317', '4318:4318'], + resource_deps=[], + labels=['observability'] + ) + +# Build Unkey binary locally (independent of infrastructure) +if start_api or start_gw or start_ctrl: + print("Building Unkey binary...") + # Build locally first for faster updates + local_resource( + 'unkey-compile', + 'CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o ./bin/unkey ./main.go', + deps=['./main.go', './pkg', './cmd', './apps'], + ) + +# API service (3 replicas) +if start_api: + print("Setting up API service...") + + docker_build_with_restart( + 'unkey-api:latest', + '.', + dockerfile='Dockerfile.tilt', + entrypoint=['/unkey', 'run', 'api'], + only=['./bin'], + live_update=[ + sync('./bin/unkey', '/unkey'), + ], + ignore=['./cmd/gw', './cmd/ctrl', './apps/gw', './apps/ctrl'] + ) + + k8s_yaml('k8s/manifests/api.yaml') + + # Build dependency list + api_deps = [] + if start_mysql: api_deps.append('mysql') + if start_clickhouse: api_deps.append('clickhouse') + if redis_started: api_deps.append('redis') + + # Add compilation dependency for Unkey services + api_deps.append('unkey-compile') + + k8s_resource( + 'api', + port_forwards='7070:7070', + resource_deps=api_deps, + labels=['unkey'], + auto_init=True, + trigger_mode=TRIGGER_MODE_AUTO if debug_mode else TRIGGER_MODE_MANUAL + ) + +# Gateway service (1 replica) +if start_gw: + print("Setting up Gateway service...") + + docker_build_with_restart( + 'unkey-gw:latest', + '.', + dockerfile='Dockerfile.tilt', + entrypoint=['/unkey', 'run', 'gw'], + only=['./bin'], + live_update=[ + sync('./bin/unkey', '/unkey'), + ], + ignore=['./cmd/api', './cmd/ctrl', './apps/api', './apps/ctrl'] + ) + + k8s_yaml('k8s/manifests/gw.yaml') + + # Build dependency list + gw_deps = [] + if start_mysql: gw_deps.append('mysql') + # Add compilation dependency for Unkey services + gw_deps.append('unkey-compile') + + k8s_resource( + 'gw', + port_forwards='6060:6060', + resource_deps=gw_deps, + labels=['unkey'], + auto_init=True, + trigger_mode=TRIGGER_MODE_AUTO if debug_mode else TRIGGER_MODE_MANUAL + ) + +# Ctrl service (1 replica) +if start_ctrl: + print("Setting up Ctrl service...") + + docker_build_with_restart( + 'unkey-ctrl:latest', + '.', + dockerfile='Dockerfile.tilt', + entrypoint=['/unkey', 'run', 'ctrl'], + only=['./bin'], + live_update=[ + sync('./bin/unkey', '/unkey'), + ], + ignore=['./cmd/api', './cmd/gw', './apps/api', './apps/gw'] + ) + + k8s_yaml('k8s/manifests/ctrl.yaml') + + # Build dependency list + ctrl_deps = [] + if start_mysql: ctrl_deps.append('mysql') + if start_s3: ctrl_deps.append('s3') + # Add compilation dependency for Unkey services + ctrl_deps.append('unkey-compile') + + k8s_resource( + 'ctrl', + port_forwards='7091:7091', + resource_deps=ctrl_deps, + labels=['unkey'], + auto_init=True, + trigger_mode=TRIGGER_MODE_AUTO if debug_mode else TRIGGER_MODE_MANUAL + ) + +# Dashboard service +if start_dashboard: + print("Setting up Dashboard service...") + # Build the dashboard image + docker_build( + 'unkey/dashboard:latest', + '../', + dockerfile='../apps/dashboard/Dockerfile', + live_update=[ + sync('../apps/dashboard', '/unkey/apps/dashboard'), + run('cd /unkey/apps/dashboard && pnpm build', trigger=['**/*.tsx', '**/*.ts', '**/*.css']), + ] + ) + k8s_yaml('k8s/manifests/dashboard.yaml') + + # Build dependency list + dashboard_deps = [] + if start_planetscale: dashboard_deps.append('planetscale') + if start_clickhouse: dashboard_deps.append('clickhouse') + + k8s_resource( + 'dashboard', + port_forwards='3000:3000', + resource_deps=dashboard_deps, + labels=['unkey'], + auto_init=True, + trigger_mode=TRIGGER_MODE_AUTO if debug_mode else TRIGGER_MODE_MANUAL + ) + +# Tilt UI customization +active_services = [] +if redis_started: active_services.append('redis') +if start_mysql: active_services.append('mysql') +if start_clickhouse: active_services.append('clickhouse') +if start_s3: active_services.append('s3') +if start_planetscale: active_services.append('planetscale') +if start_observability: active_services.extend(['prometheus', 'otel-collector']) +if start_api: active_services.append('api') +if start_gw: active_services.append('gw') +if start_ctrl: active_services.append('ctrl') +if start_dashboard: active_services.append('dashboard') + +print(""" +Tilt is ready! + +Services running: %s + +Web UI: http://localhost:10350 + +Services available via Tilt port forwards: +Dashboard: http://localhost:3000 +API: http://localhost:7070 +Gateway: http://localhost:6060 +Ctrl: http://localhost:7091 +Prometheus: http://localhost:9090 +S3 Console: http://localhost:9000 +ClickHouse: http://localhost:8123 + +Tips: +- Press 's' to open a shell in running containers +- Press 'r' to manually trigger rebuilds +- Use 'tilt down' to stop all services +- Use 'tilt logs ' to view specific service logs + +Development workflow: +- Edit Go files -> automatic rebuild & restart +- View logs in unified Tilt UI +- Use ingress for service access + +""" % ', '.join(active_services)) \ No newline at end of file diff --git a/go/k8s/manifests/api.yaml b/go/k8s/manifests/api.yaml new file mode 100644 index 0000000000..e2dca4ff92 --- /dev/null +++ b/go/k8s/manifests/api.yaml @@ -0,0 +1,113 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: api + namespace: unkey + labels: + app: api +spec: + replicas: 3 + selector: + matchLabels: + app: api + template: + metadata: + labels: + app: api + spec: + containers: + - name: api + image: unkey-api:latest + imagePullPolicy: Never # Use local images + ports: + - containerPort: 7070 + env: + # Server Configuration + - name: UNKEY_HTTP_PORT + value: "7070" + - name: UNKEY_LOGS_COLOR + value: "true" + - name: UNKEY_TEST_MODE + value: "false" + # Instance Identification + - name: UNKEY_PLATFORM + value: "kubernetes" + - name: UNKEY_IMAGE + value: "unkey:latest" + - name: UNKEY_REGION + value: "local" + - name: UNKEY_INSTANCE_ID + value: "api-dev" + # Database Configuration + - name: UNKEY_DATABASE_PRIMARY + value: "unkey:password@tcp(mysql:3306)/unkey?parseTime=true&interpolateParams=true" + # Caching and Storage + - name: UNKEY_REDIS_URL + value: "redis://redis:6379" + - name: UNKEY_CLICKHOUSE_URL + value: "clickhouse://unkey:password@clickhouse:9000?secure=false&skip_verify=true" + # Observability - DISABLED for development + - name: UNKEY_OTEL + value: "false" + - name: UNKEY_PROMETHEUS_PORT + value: "0" + # Vault Configuration + - name: UNKEY_VAULT_MASTER_KEYS + value: "Ch9rZWtfMmdqMFBJdVhac1NSa0ZhNE5mOWlLSnBHenFPENTt7an5MRogENt9Si6wms4pQ2XIvqNSIgNpaBenJmXgcInhu6Nfv2U=" + - name: UNKEY_VAULT_S3_URL + value: "http://s3:9000" + - name: UNKEY_VAULT_S3_BUCKET + value: "vault" + - name: UNKEY_VAULT_S3_ACCESS_KEY_ID + value: "minio_root_user" + - name: UNKEY_VAULT_S3_ACCESS_KEY_SECRET + value: "minio_root_password" + # ClickHouse Proxy Service Configuration + - name: UNKEY_CHPROXY_AUTH_TOKEN + value: "chproxy-test-token-123" + # Request Body Configuration + - name: UNKEY_MAX_REQUEST_BODY_SIZE + value: "10485760" + command: ["/unkey", "run", "api"] + readinessProbe: + httpGet: + path: /v2/liveness + port: 7070 + initialDelaySeconds: 10 + periodSeconds: 5 + livenessProbe: + httpGet: + path: /v2/liveness + port: 7070 + initialDelaySeconds: 30 + periodSeconds: 10 + resources: + requests: + memory: "256Mi" + cpu: "200m" + limits: + memory: "1Gi" + cpu: "1000m" + initContainers: + - name: wait-for-dependencies + image: busybox:1.36 + command: ['sh', '-c', 'until nc -z mysql 3306 && nc -z clickhouse 8123; do echo waiting for dependencies; sleep 2; done;'] + +--- +apiVersion: v1 +kind: Service +metadata: + name: api + namespace: unkey + labels: + app: api +spec: + selector: + app: api + ports: + - name: http + port: 7070 + targetPort: 7070 + protocol: TCP + type: NodePort \ No newline at end of file diff --git a/go/k8s/manifests/clickhouse.yaml b/go/k8s/manifests/clickhouse.yaml new file mode 100644 index 0000000000..4a06255023 --- /dev/null +++ b/go/k8s/manifests/clickhouse.yaml @@ -0,0 +1,94 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: clickhouse-pvc + namespace: unkey +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: clickhouse + namespace: unkey + labels: + app: clickhouse +spec: + replicas: 1 + selector: + matchLabels: + app: clickhouse + template: + metadata: + labels: + app: clickhouse + spec: + containers: + - name: clickhouse + image: clickhouse/clickhouse-server:24.3 + ports: + - containerPort: 8123 + name: http + - containerPort: 9000 + name: native + env: + - name: CLICKHOUSE_DB + value: "unkey" + - name: CLICKHOUSE_USER + value: "unkey" + - name: CLICKHOUSE_PASSWORD + value: "password" + volumeMounts: + - name: clickhouse-storage + mountPath: /var/lib/clickhouse + readinessProbe: + httpGet: + path: /ping + port: 8123 + initialDelaySeconds: 30 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /ping + port: 8123 + initialDelaySeconds: 60 + periodSeconds: 30 + resources: + requests: + memory: "256Mi" + cpu: "200m" + limits: + memory: "1Gi" + cpu: "1000m" + volumes: + - name: clickhouse-storage + persistentVolumeClaim: + claimName: clickhouse-pvc + +--- +apiVersion: v1 +kind: Service +metadata: + name: clickhouse + namespace: unkey + labels: + app: clickhouse +spec: + selector: + app: clickhouse + ports: + - name: http + port: 8123 + targetPort: 8123 + protocol: TCP + - name: native + port: 9000 + targetPort: 9000 + protocol: TCP + type: ClusterIP \ No newline at end of file diff --git a/go/k8s/manifests/ctrl.yaml b/go/k8s/manifests/ctrl.yaml new file mode 100644 index 0000000000..8f34d1af9e --- /dev/null +++ b/go/k8s/manifests/ctrl.yaml @@ -0,0 +1,104 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ctrl + namespace: unkey + labels: + app: ctrl +spec: + replicas: 1 + selector: + matchLabels: + app: ctrl + template: + metadata: + labels: + app: ctrl + spec: + serviceAccountName: ctrl-service-account + containers: + - name: ctrl + image: unkey-ctrl:latest + imagePullPolicy: Never # Use local images + ports: + - containerPort: 7091 + env: + # Server Configuration + - name: UNKEY_HTTP_PORT + value: "7091" + - name: UNKEY_LOGS_COLOR + value: "true" + # Instance Identification + - name: UNKEY_PLATFORM + value: "kubernetes" + - name: UNKEY_IMAGE + value: "unkey:latest" + - name: UNKEY_REGION + value: "local" + - name: UNKEY_INSTANCE_ID + value: "ctrl-dev" + # Database Configuration + - name: UNKEY_DATABASE_PRIMARY + value: "unkey:password@tcp(mysql:3306)/unkey?parseTime=true&interpolateParams=true" + - name: UNKEY_DATABASE_PARTITION + value: "unkey:password@tcp(mysql:3306)/partition_001?parseTime=true&interpolateParams=true" + - name: UNKEY_DATABASE_HYDRA + value: "unkey:password@tcp(mysql:3306)/hydra?parseTime=true&interpolateParams=true" + # Observability - DISABLED for development + - name: UNKEY_OTEL + value: "false" + # Control Plane Specific + - name: UNKEY_AUTH_TOKEN + value: "dev-auth-token" + - name: UNKEY_METALD_ADDRESS + value: "http://metald-placeholder:8080" + - name: UNKEY_SPIFFE_SOCKET_PATH + value: "/var/lib/spire/agent/agent.sock" + # Vault Configuration (required) + - name: UNKEY_VAULT_MASTER_KEYS + value: "Ch9rZWtfMmdqMFBJdVhac1NSa0ZhNE5mOWlLSnBHenFPENTt7an5MRogENt9Si6wms4pQ2XIvqNSIgNpaBenJmXgcInhu6Nfv2U=" + - name: UNKEY_VAULT_S3_URL + value: "http://s3:9000" + - name: UNKEY_VAULT_S3_BUCKET + value: "vault" + - name: UNKEY_VAULT_S3_ACCESS_KEY_ID + value: "minio_root_user" + - name: UNKEY_VAULT_S3_ACCESS_KEY_SECRET + value: "minio_root_password" + # Additional Configuration + - name: UNKEY_ACME_ENABLED + value: "false" + - name: UNKEY_METALD_FALLBACK + value: "k8s" + command: ["/unkey", "run", "ctrl"] + resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "512Mi" + cpu: "500m" + initContainers: + - name: wait-for-dependencies + image: busybox:1.36 + command: ['sh', '-c', 'until nc -z mysql 3306 && nc -z s3 9000; do echo waiting for dependencies; sleep 2; done;'] + +--- +apiVersion: v1 +kind: Service +metadata: + name: ctrl + namespace: unkey + labels: + app: ctrl +spec: + selector: + app: ctrl + ports: + - name: http + port: 7091 + targetPort: 7091 + protocol: TCP + type: NodePort + diff --git a/go/k8s/manifests/dashboard.yaml b/go/k8s/manifests/dashboard.yaml new file mode 100644 index 0000000000..317962de83 --- /dev/null +++ b/go/k8s/manifests/dashboard.yaml @@ -0,0 +1,87 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: dashboard + namespace: unkey + labels: + app: dashboard +spec: + replicas: 1 + selector: + matchLabels: + app: dashboard + template: + metadata: + labels: + app: dashboard + spec: + initContainers: + - name: wait-for-dependencies + image: busybox:1.36 + command: + - sh + - -c + - | + until nc -z planetscale 3900; do + echo waiting for planetscale + sleep 2 + done + containers: + - name: dashboard + image: unkey/dashboard:latest + imagePullPolicy: Never + ports: + - containerPort: 3000 + env: + # Database configuration + - name: DATABASE_HOST + value: "planetscale:3900" + # ClickHouse configuration + - name: CLICKHOUSE_URL + value: "http://unkey:password@clickhouse:8123" + # Environment + - name: NODE_ENV + value: "production" + # Instance identification + - name: UNKEY_PLATFORM + value: "kubernetes" + - name: UNKEY_REGION + value: "local" + readinessProbe: + httpGet: + path: / + port: 3000 + initialDelaySeconds: 10 + periodSeconds: 5 + livenessProbe: + httpGet: + path: / + port: 3000 + initialDelaySeconds: 30 + periodSeconds: 10 + resources: + requests: + memory: "256Mi" + cpu: "200m" + limits: + memory: "512Mi" + cpu: "500m" + +--- +apiVersion: v1 +kind: Service +metadata: + name: dashboard + namespace: unkey + labels: + app: dashboard +spec: + selector: + app: dashboard + ports: + - name: http + port: 3000 + targetPort: 3000 + protocol: TCP + type: NodePort \ No newline at end of file diff --git a/go/k8s/manifests/gw.yaml b/go/k8s/manifests/gw.yaml new file mode 100644 index 0000000000..8511d64f6d --- /dev/null +++ b/go/k8s/manifests/gw.yaml @@ -0,0 +1,122 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: gw + namespace: unkey + labels: + app: gw +spec: + replicas: 1 + selector: + matchLabels: + app: gw + template: + metadata: + labels: + app: gw + spec: + containers: + - name: gw + image: unkey-gw:latest + imagePullPolicy: Never # Use local images + ports: + - containerPort: 6060 + - containerPort: 6443 + env: + # Server Configuration + - name: UNKEY_HTTP_PORT + value: "6060" + - name: UNKEY_HTTPS_PORT + value: "6443" + - name: UNKEY_TLS_ENABLED + value: "false" + # Instance Identification + - name: UNKEY_PLATFORM + value: "kubernetes" + - name: UNKEY_IMAGE + value: "unkey:latest" + - name: UNKEY_REGION + value: "local" + - name: UNKEY_GATEWAY_ID + value: "gw-dev" + - name: UNKEY_DEFAULT_CERT_DOMAIN + value: "" + - name: UNKEY_MAIN_DOMAIN + value: "" + - name: UNKEY_CTRL_ADDR + value: "ctrl:7091" + # Database Configuration - Partitioned (for gateway operations) + - name: UNKEY_DATABASE_PRIMARY + value: "unkey:password@tcp(mysql:3306)/partition_001?parseTime=true&interpolateParams=true" + # Database Configuration - Keys Service + - name: UNKEY_KEYS_DATABASE_PRIMARY + value: "unkey:password@tcp(mysql:3306)/unkey?parseTime=true&interpolateParams=true" + # ClickHouse Configuration + - name: UNKEY_CLICKHOUSE_URL + value: "clickhouse://unkey:password@clickhouse:9000?secure=false&skip_verify=true" + # Redis Configuration + - name: UNKEY_REDIS_URL + value: "redis://redis:6379" + # Observability - DISABLED for development + - name: UNKEY_OTEL + value: "false" + - name: UNKEY_PROMETHEUS_PORT + value: "0" + # Vault Configuration + - name: UNKEY_VAULT_MASTER_KEYS + value: "Ch9rZWtfMmdqMFBJdVhac1NSa0ZhNE5mOWlLSnBHenFPENTt7an5MRogENt9Si6wms4pQ2XIvqNSIgNpaBenJmXgcInhu6Nfv2U=" + - name: UNKEY_VAULT_S3_URL + value: "http://s3:9000" + - name: UNKEY_VAULT_S3_BUCKET + value: "vault" + - name: UNKEY_VAULT_S3_ACCESS_KEY_ID + value: "minio_root_user" + - name: UNKEY_VAULT_S3_ACCESS_KEY_SECRET + value: "minio_root_password" + command: ["/unkey", "run", "gw"] + readinessProbe: + httpGet: + path: /unkey/_internal/liveness + port: 6060 + initialDelaySeconds: 10 + periodSeconds: 5 + livenessProbe: + httpGet: + path: /unkey/_internal/liveness + port: 6060 + initialDelaySeconds: 30 + periodSeconds: 10 + resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "512Mi" + cpu: "500m" + initContainers: + - name: wait-for-dependencies + image: busybox:1.36 + command: ['sh', '-c', 'until nc -z mysql 3306; do echo waiting for dependencies; sleep 2; done;'] + +--- +apiVersion: v1 +kind: Service +metadata: + name: gw + namespace: unkey + labels: + app: gw +spec: + selector: + app: gw + ports: + - name: http + port: 6060 + targetPort: 6060 + protocol: TCP + - name: https + port: 6443 + targetPort: 6443 + protocol: TCP + type: NodePort \ No newline at end of file diff --git a/go/k8s/manifests/ingress.yaml b/go/k8s/manifests/ingress.yaml new file mode 100644 index 0000000000..a495d63b67 --- /dev/null +++ b/go/k8s/manifests/ingress.yaml @@ -0,0 +1,2 @@ +# OrbStack handles LoadBalancer services automatically +# Services are accessible at .k8s.orb.local \ No newline at end of file diff --git a/go/k8s/manifests/mysql.yaml b/go/k8s/manifests/mysql.yaml new file mode 100644 index 0000000000..e3c2e18d51 --- /dev/null +++ b/go/k8s/manifests/mysql.yaml @@ -0,0 +1,90 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: mysql-pvc + namespace: unkey +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mysql + namespace: unkey + labels: + app: mysql +spec: + replicas: 1 + selector: + matchLabels: + app: mysql + template: + metadata: + labels: + app: mysql + spec: + containers: + - name: mysql + image: unkey/mysql:latest + imagePullPolicy: Never + ports: + - containerPort: 3306 + env: + - name: MYSQL_ROOT_PASSWORD + value: "root" + - name: MYSQL_DATABASE + value: "unkey" + - name: MYSQL_USER + value: "unkey" + - name: MYSQL_PASSWORD + value: "password" + args: + - "--max_connections=1000" + volumeMounts: + - name: mysql-storage + mountPath: /var/lib/mysql + readinessProbe: + exec: + command: + - mysqladmin + - ping + - -h + - localhost + initialDelaySeconds: 30 + periodSeconds: 10 + livenessProbe: + exec: + command: + - mysqladmin + - ping + - -h + - localhost + initialDelaySeconds: 60 + periodSeconds: 30 + volumes: + - name: mysql-storage + persistentVolumeClaim: + claimName: mysql-pvc + +--- +apiVersion: v1 +kind: Service +metadata: + name: mysql + namespace: unkey + labels: + app: mysql +spec: + selector: + app: mysql + ports: + - port: 3306 + targetPort: 3306 + protocol: TCP + type: ClusterIP \ No newline at end of file diff --git a/go/k8s/manifests/namespace.yaml b/go/k8s/manifests/namespace.yaml new file mode 100644 index 0000000000..192d856504 --- /dev/null +++ b/go/k8s/manifests/namespace.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: unkey + labels: + app.kubernetes.io/name: unkey + app.kubernetes.io/instance: dev \ No newline at end of file diff --git a/go/k8s/manifests/observability.yaml b/go/k8s/manifests/observability.yaml new file mode 100644 index 0000000000..d3fcd0039d --- /dev/null +++ b/go/k8s/manifests/observability.yaml @@ -0,0 +1,197 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-config + namespace: unkey +data: + prometheus.yml: | + global: + scrape_interval: 15s + scrape_configs: + - job_name: 'unkey-services' + static_configs: + - targets: ['ctrl:8084'] + metrics_path: /metrics + scrape_interval: 5s + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: prometheus + namespace: unkey + labels: + app: prometheus +spec: + replicas: 1 + selector: + matchLabels: + app: prometheus + template: + metadata: + labels: + app: prometheus + spec: + containers: + - name: prometheus + image: prom/prometheus:v2.45.0 + ports: + - containerPort: 9090 + args: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus/' + - '--web.console.libraries=/etc/prometheus/console_libraries' + - '--web.console.templates=/etc/prometheus/consoles' + - '--storage.tsdb.retention.time=200h' + - '--web.enable-lifecycle' + volumeMounts: + - name: prometheus-config + mountPath: /etc/prometheus/ + resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "512Mi" + cpu: "500m" + volumes: + - name: prometheus-config + configMap: + name: prometheus-config + +--- +apiVersion: v1 +kind: Service +metadata: + name: prometheus + namespace: unkey + labels: + app: prometheus +spec: + selector: + app: prometheus + ports: + - port: 9090 + targetPort: 9090 + protocol: TCP + type: NodePort + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: otel-collector + namespace: unkey + labels: + app: otel-collector +spec: + replicas: 1 + selector: + matchLabels: + app: otel-collector + template: + metadata: + labels: + app: otel-collector + spec: + containers: + - name: otel-collector + image: otel/opentelemetry-collector-contrib:0.92.0 + ports: + - containerPort: 4317 # OTLP gRPC + - containerPort: 4318 # OTLP HTTP + - containerPort: 8889 # Prometheus metrics + command: + - /otelcol-contrib + - --config=/etc/otelcol-contrib/otel-collector.yml + env: + - name: GOMEMLIMIT + value: "160MiB" + volumeMounts: + - name: otel-collector-config + mountPath: /etc/otelcol-contrib/ + resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "256Mi" + cpu: "200m" + volumes: + - name: otel-collector-config + configMap: + name: otel-collector-config + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: otel-collector-config + namespace: unkey +data: + otel-collector.yml: | + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + prometheus: + config: + scrape_configs: + - job_name: 'otel-collector' + scrape_interval: 10s + static_configs: + - targets: ['0.0.0.0:8889'] + + processors: + batch: + + exporters: + prometheus: + endpoint: "0.0.0.0:8889" + logging: + loglevel: info + + service: + pipelines: + traces: + receivers: [otlp] + processors: [batch] + exporters: [logging] + metrics: + receivers: [otlp, prometheus] + processors: [batch] + exporters: [prometheus, logging] + logs: + receivers: [otlp] + processors: [batch] + exporters: [logging] + +--- +apiVersion: v1 +kind: Service +metadata: + name: otel-collector + namespace: unkey + labels: + app: otel-collector +spec: + selector: + app: otel-collector + ports: + - name: otlp-grpc + port: 4317 + targetPort: 4317 + protocol: TCP + - name: otlp-http + port: 4318 + targetPort: 4318 + protocol: TCP + - name: prometheus + port: 8889 + targetPort: 8889 + protocol: TCP + type: NodePort \ No newline at end of file diff --git a/go/k8s/manifests/planetscale.yaml b/go/k8s/manifests/planetscale.yaml new file mode 100644 index 0000000000..a3551d3eb1 --- /dev/null +++ b/go/k8s/manifests/planetscale.yaml @@ -0,0 +1,67 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: planetscale + namespace: unkey + labels: + app: planetscale +spec: + replicas: 1 + selector: + matchLabels: + app: planetscale + template: + metadata: + labels: + app: planetscale + spec: + containers: + - name: planetscale-http + image: ghcr.io/mattrobenolt/ps-http-sim:v0.0.12 + ports: + - containerPort: 3900 + args: ["-listen-port=3900", "-mysql-dbname=unkey", "-mysql-addr=mysql", "-mysql-max-rows=100000", "-mysql-idle-timeout=1s"] + env: + - name: DATABASE_HOST + value: "mysql" + - name: DATABASE_NAME + value: "unkey" + - name: DATABASE_USERNAME + value: "unkey" + - name: DATABASE_PASSWORD + value: "password" + readinessProbe: + tcpSocket: + port: 3900 + initialDelaySeconds: 10 + periodSeconds: 5 + livenessProbe: + tcpSocket: + port: 3900 + initialDelaySeconds: 30 + periodSeconds: 10 + resources: + requests: + memory: "64Mi" + cpu: "50m" + limits: + memory: "256Mi" + cpu: "200m" + +--- +apiVersion: v1 +kind: Service +metadata: + name: planetscale + namespace: unkey + labels: + app: planetscale +spec: + selector: + app: planetscale + ports: + - port: 3900 + targetPort: 3900 + protocol: TCP + type: ClusterIP \ No newline at end of file diff --git a/go/k8s/manifests/rbac.yaml b/go/k8s/manifests/rbac.yaml new file mode 100644 index 0000000000..074232ace7 --- /dev/null +++ b/go/k8s/manifests/rbac.yaml @@ -0,0 +1,42 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ctrl-service-account + namespace: unkey + labels: + app: ctrl + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: ctrl-deployment-manager + labels: + app: ctrl +rules: +- apiGroups: ["apps"] + resources: ["deployments"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +- apiGroups: [""] + resources: ["pods", "services"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +- apiGroups: [""] + resources: ["configmaps", "secrets"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: ctrl-deployment-manager-binding + labels: + app: ctrl +subjects: +- kind: ServiceAccount + name: ctrl-service-account + namespace: unkey +roleRef: + kind: ClusterRole + name: ctrl-deployment-manager + apiGroup: rbac.authorization.k8s.io \ No newline at end of file diff --git a/go/k8s/manifests/redis.yaml b/go/k8s/manifests/redis.yaml new file mode 100644 index 0000000000..f843f09911 --- /dev/null +++ b/go/k8s/manifests/redis.yaml @@ -0,0 +1,61 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: redis + namespace: unkey + labels: + app: redis +spec: + replicas: 1 + selector: + matchLabels: + app: redis + template: + metadata: + labels: + app: redis + spec: + containers: + - name: redis + image: redis:8.0 + ports: + - containerPort: 6379 + readinessProbe: + exec: + command: + - redis-cli + - ping + initialDelaySeconds: 5 + periodSeconds: 5 + livenessProbe: + exec: + command: + - redis-cli + - ping + initialDelaySeconds: 10 + periodSeconds: 30 + resources: + requests: + memory: "64Mi" + cpu: "50m" + limits: + memory: "256Mi" + cpu: "200m" + +--- +apiVersion: v1 +kind: Service +metadata: + name: redis + namespace: unkey + labels: + app: redis +spec: + selector: + app: redis + ports: + - port: 6379 + targetPort: 6379 + protocol: TCP + type: ClusterIP \ No newline at end of file diff --git a/go/k8s/manifests/s3.yaml b/go/k8s/manifests/s3.yaml new file mode 100644 index 0000000000..e0aeead721 --- /dev/null +++ b/go/k8s/manifests/s3.yaml @@ -0,0 +1,113 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: s3-pvc + namespace: unkey +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: s3 + namespace: unkey + labels: + app: s3 +spec: + replicas: 1 + selector: + matchLabels: + app: s3 + template: + metadata: + labels: + app: s3 + spec: + containers: + - name: minio + image: bitnami/minio:2025.7.23 + ports: + - containerPort: 9000 + name: api + - containerPort: 9001 + name: console + env: + - name: MINIO_ROOT_USER + value: "minio_root_user" + - name: MINIO_ROOT_PASSWORD + value: "minio_root_password" + - name: MINIO_API_PORT_NUMBER + value: "9000" + - name: MINIO_CONSOLE_PORT_NUMBER + value: "9001" + - name: MINIO_DEFAULT_BUCKETS + value: "vault" + volumeMounts: + - name: s3-storage + mountPath: /data + readinessProbe: + httpGet: + path: /minio/health/ready + port: 9000 + initialDelaySeconds: 30 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /minio/health/live + port: 9000 + initialDelaySeconds: 60 + periodSeconds: 30 + resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "512Mi" + cpu: "500m" + volumes: + - name: s3-storage + persistentVolumeClaim: + claimName: s3-pvc + +--- +apiVersion: v1 +kind: Service +metadata: + name: s3 + namespace: unkey + labels: + app: s3 +spec: + selector: + app: s3 + ports: + - name: api + port: 9000 + targetPort: 9000 + protocol: TCP + type: NodePort + +--- +apiVersion: v1 +kind: Service +metadata: + name: s3-console + namespace: unkey + labels: + app: s3 + service: console +spec: + selector: + app: s3 + ports: + - name: console + port: 9001 + targetPort: 9001 + protocol: TCP + type: NodePort \ No newline at end of file From 1c33c01637e5ba1d34a1a684fab43aa4289d0bf8 Mon Sep 17 00:00:00 2001 From: Flo Date: Wed, 10 Sep 2025 21:40:27 +0200 Subject: [PATCH 2/9] chore(setup): try out lb --- go/Dockerfile | 2 -- go/K8S_DEVELOPMENT.md | 60 ++++++++++++++++++++++++++++----------- go/k8s/manifests/api.yaml | 21 +++++++++++++- go/k8s/manifests/gw.yaml | 25 +++++++++++++++- 4 files changed, 87 insertions(+), 21 deletions(-) diff --git a/go/Dockerfile b/go/Dockerfile index 5fa975b02b..c025d32077 100644 --- a/go/Dockerfile +++ b/go/Dockerfile @@ -13,8 +13,6 @@ RUN go build -o bin/unkey -ldflags="-X 'github.com/unkeyed/unkey/go/pkg/version. FROM ${BASE_IMAGE} -# When BASE_IMAGE is golang-alpine (for Tilt dev), Go tools are already available - COPY --from=builder /go/src/github.com/unkeyed/unkey/go/bin/unkey / LABEL org.opencontainers.image.source=https://github.com/unkeyed/unkey/go diff --git a/go/K8S_DEVELOPMENT.md b/go/K8S_DEVELOPMENT.md index 9e5e7daff4..e6d81c8eb6 100644 --- a/go/K8S_DEVELOPMENT.md +++ b/go/K8S_DEVELOPMENT.md @@ -10,32 +10,37 @@ Install the required tools: - **kubectl**: https://kubernetes.io/docs/tasks/tools/ Or check if already installed: + ```bash make k8s-check ``` Optional for enhanced development experience: + - **Tilt**: https://docs.tilt.dev/install.html ## Quick Start ### Option 1: Full Environment (Recommended) + ```bash make k8s-up ``` This will: + - Use your current Kubernetes cluster (Docker Desktop/OrbStack) - Build and deploy all services (MySQL, ClickHouse, S3, Observability, Unkey) - Wait for all services to be ready - Show connection info ### Option 2: Individual Services + ```bash # Start only MySQL make start-mysql -# Start only ctrl (requires cluster to exist) +# Start only ctrl (requires cluster to exist) make start-ctrl # Start all services individually @@ -43,11 +48,13 @@ make start-all ``` ### Option 3: Enhanced Development with Tilt + ```bash make dev ``` If Tilt is installed, this provides: + - Hot reloading for Go code changes - Unified log viewing in web UI (http://localhost:10350) - Resource management dashboard @@ -65,8 +72,9 @@ make k8s-ports ``` This will show the randomly assigned NodePorts, e.g.: -- **Dashboard**: http://localhost:32001 -- **API**: http://localhost:32002 + +- **Dashboard**: http://localhost:3000 +- **API**: http://localhost:32002 - **Gateway**: http://localhost:32003 - **Ctrl**: http://localhost:32004 - **Prometheus**: http://localhost:32005 @@ -77,6 +85,7 @@ This will show the randomly assigned NodePorts, e.g.: OrbStack also supports LoadBalancer services with automatic `*.k8s.orb.local` domains, but NodePort is simpler for development. ### Inside the cluster + - **MySQL**: `mysql.unkey.svc.cluster.local:3306` - **ClickHouse**: `clickhouse.unkey.svc.cluster.local:8123` - **S3**: `s3.unkey.svc.cluster.local:9000` @@ -88,12 +97,14 @@ OrbStack also supports LoadBalancer services with automatic `*.k8s.orb.local` do ### Making Code Changes #### With Tilt (Hot Reloading) + 1. Start Tilt: `make dev` 2. Edit Go files 3. Changes automatically rebuild and restart services 4. View logs in Tilt UI #### Without Tilt (Manual) + 1. Make code changes 2. Rebuild and redeploy: ```bash @@ -132,14 +143,18 @@ kubectl describe pod -n unkey ## Configuration ### Environment Variables + The ctrl service uses these key environment variables: + - `UNKEY_DATABASE_PRIMARY`: Connection to partition database -- `UNKEY_DATABASE_HYDRA`: Connection to hydra database +- `UNKEY_DATABASE_HYDRA`: Connection to hydra database - `UNKEY_HTTP_PORT`: Service port (8084) - `UNKEY_PLATFORM`: Set to "kubernetes" ### Database Setup + MySQL is automatically configured with: + - `unkey` database for main data - `hydra` database for OAuth flows - `partition_001` database for partitioned data @@ -150,11 +165,12 @@ All schemas are automatically applied on startup. ## Customization ### Selective Services with Tilt + ```bash # Start only database services tilt up mysql,clickhouse,planetscale -# Start only storage services +# Start only storage services tilt up s3 # Start only observability @@ -165,7 +181,9 @@ tilt up ctrl ``` ### Custom Resource Limits + Edit `k8s/manifests/ctrl.yaml` to adjust: + ```yaml resources: requests: @@ -177,7 +195,9 @@ resources: ``` ### Persistent Data + MySQL data persists between restarts via PersistentVolumeClaim. To reset: + ```bash kubectl delete pvc -n unkey mysql-pvc make start-mysql @@ -188,6 +208,7 @@ make start-mysql ### Common Issues #### Cluster Not Available + ```bash # Make sure Kubernetes is enabled in Docker Desktop/OrbStack # Check current context @@ -198,6 +219,7 @@ kubectl config use-context docker-desktop # or orbstack ``` #### Services Not Ready + ```bash # Check events kubectl get events -n unkey --sort-by='.lastTimestamp' @@ -208,7 +230,9 @@ kubectl logs -n unkey -l app=ctrl ``` #### Port Conflicts + If ports 8084 or 3306 are in use: + ```bash # Check what's using the port lsof -i :8084 @@ -218,6 +242,7 @@ lsof -i :3306 ``` #### Image Build Issues + ```bash # Manual build docker build -t unkey/mysql:latest -f ../deployment/Dockerfile.mysql ../ @@ -225,22 +250,23 @@ docker build -t unkey:latest . ``` ### Performance Tips + - Use `make dev` for fastest development cycle - Keep cluster running between sessions (don't run `k8s-down`) - Use selective service startup when working on specific components ## vs Docker Compose -| Feature | Kubernetes | Docker Compose | -|---------|-----------|----------------| -| **Prod Similarity** | ✅ Real Kubernetes | ❌ Different from prod | -| **Resource Usage** | ✅ Native containers | ✅ Native containers | -| **Setup Complexity** | ⚠️ Enable K8s in Docker/OrbStack | ✅ Simple setup | -| **Service Discovery** | ✅ K8s native DNS | ⚠️ Docker networks | -| **Scaling** | ✅ Easy horizontal scaling | ❌ Limited scaling | -| **Hot Reloading** | ✅ With Tilt | ⚠️ Manual restarts | -| **Debugging** | ✅ Rich tooling | ✅ Simple logs | -| **Architecture** | ✅ Clean separation | ✅ Clean separation | +| Feature | Kubernetes | Docker Compose | +| --------------------- | -------------------------------- | ---------------------- | +| **Prod Similarity** | ✅ Real Kubernetes | ❌ Different from prod | +| **Resource Usage** | ✅ Native containers | ✅ Native containers | +| **Setup Complexity** | ⚠️ Enable K8s in Docker/OrbStack | ✅ Simple setup | +| **Service Discovery** | ✅ K8s native DNS | ⚠️ Docker networks | +| **Scaling** | ✅ Easy horizontal scaling | ❌ Limited scaling | +| **Hot Reloading** | ✅ With Tilt | ⚠️ Manual restarts | +| **Debugging** | ✅ Rich tooling | ✅ Simple logs | +| **Architecture** | ✅ Clean separation | ✅ Clean separation | Choose Kubernetes for true prod-like development, Docker Compose for quick iterations. @@ -249,10 +275,10 @@ Choose Kubernetes for true prod-like development, Docker Compose for quick itera All services can be started individually or together: - `mysql` - MySQL database with schemas -- `clickhouse` - ClickHouse analytics database +- `clickhouse` - ClickHouse analytics database - `s3` - MinIO object storage - `planetscale` - PlanetScale HTTP database proxy - `observability` - Prometheus + OTEL Collector - `ctrl` - Main Unkey service (ctrl/api/gw) -Use `make k8s-up` to start everything or selectively with Tilt. \ No newline at end of file +Use `make k8s-up` to start everything or selectively with Tilt. diff --git a/go/k8s/manifests/api.yaml b/go/k8s/manifests/api.yaml index e2dca4ff92..6f263e9c5c 100644 --- a/go/k8s/manifests/api.yaml +++ b/go/k8s/manifests/api.yaml @@ -110,4 +110,23 @@ spec: port: 7070 targetPort: 7070 protocol: TCP - type: NodePort \ No newline at end of file + type: NodePort + +--- +apiVersion: v1 +kind: Service +metadata: + name: api-lb + namespace: unkey + labels: + app: api + service: loadbalancer +spec: + selector: + app: api + ports: + - name: http + port: 80 + targetPort: 7070 + protocol: TCP + type: LoadBalancer \ No newline at end of file diff --git a/go/k8s/manifests/gw.yaml b/go/k8s/manifests/gw.yaml index 8511d64f6d..912258a395 100644 --- a/go/k8s/manifests/gw.yaml +++ b/go/k8s/manifests/gw.yaml @@ -119,4 +119,27 @@ spec: port: 6443 targetPort: 6443 protocol: TCP - type: NodePort \ No newline at end of file + type: NodePort + +--- +apiVersion: v1 +kind: Service +metadata: + name: gw-lb + namespace: unkey + labels: + app: gw + service: loadbalancer +spec: + selector: + app: gw + ports: + - name: http + port: 80 + targetPort: 6060 + protocol: TCP + - name: https + port: 443 + targetPort: 6443 + protocol: TCP + type: LoadBalancer \ No newline at end of file From 1bdae1ef27fa0ba71f0dfa5963967d18dc1ac6f1 Mon Sep 17 00:00:00 2001 From: Flo Date: Wed, 10 Sep 2025 23:50:35 +0200 Subject: [PATCH 3/9] merge some changes --- go/Tiltfile | 45 +++--- go/k8s/manifests/api.yaml | 173 +++++++++++------------ go/k8s/manifests/clickhouse.yaml | 96 ++++++------- go/k8s/manifests/ctrl.yaml | 142 +++++++++---------- go/k8s/manifests/dashboard.yaml | 108 +++++++-------- go/k8s/manifests/gw.yaml | 206 +++++++++++++--------------- go/k8s/manifests/ingress.yaml | 2 - go/k8s/manifests/mysql.yaml | 88 ++++++------ go/k8s/manifests/namespace.yaml | 2 +- go/k8s/manifests/observability.yaml | 130 +++++++++--------- go/k8s/manifests/planetscale.yaml | 77 ++++++----- go/k8s/manifests/rbac.yaml | 26 ++-- go/k8s/manifests/redis.yaml | 58 ++++---- go/k8s/manifests/s3.yaml | 104 +++++++------- 14 files changed, 626 insertions(+), 631 deletions(-) delete mode 100644 go/k8s/manifests/ingress.yaml diff --git a/go/Tiltfile b/go/Tiltfile index 5c0ca9df1f..7ef4fa545c 100644 --- a/go/Tiltfile +++ b/go/Tiltfile @@ -29,9 +29,8 @@ start_gw = 'all' in services or 'gateway' in services or 'gw' in services start_ctrl = 'all' in services or 'ctrl' in services start_dashboard = 'all' in services or 'dashboard' in services -# Apply RBAC and ingress +# Apply RBAC k8s_yaml('k8s/manifests/rbac.yaml') -k8s_yaml('k8s/manifests/ingress.yaml') # Redis service redis_started = False @@ -130,7 +129,7 @@ if start_api or start_gw or start_ctrl: # API service (3 replicas) if start_api: print("Setting up API service...") - + docker_build_with_restart( 'unkey-api:latest', '.', @@ -142,18 +141,18 @@ if start_api: ], ignore=['./cmd/gw', './cmd/ctrl', './apps/gw', './apps/ctrl'] ) - + k8s_yaml('k8s/manifests/api.yaml') - + # Build dependency list api_deps = [] if start_mysql: api_deps.append('mysql') if start_clickhouse: api_deps.append('clickhouse') - if redis_started: api_deps.append('redis') - + if redis_started: api_deps.append('redis') + # Add compilation dependency for Unkey services api_deps.append('unkey-compile') - + k8s_resource( 'api', port_forwards='7070:7070', @@ -163,10 +162,10 @@ if start_api: trigger_mode=TRIGGER_MODE_AUTO if debug_mode else TRIGGER_MODE_MANUAL ) -# Gateway service (1 replica) +# Gateway service (1 replica) if start_gw: print("Setting up Gateway service...") - + docker_build_with_restart( 'unkey-gw:latest', '.', @@ -178,15 +177,15 @@ if start_gw: ], ignore=['./cmd/api', './cmd/ctrl', './apps/api', './apps/ctrl'] ) - + k8s_yaml('k8s/manifests/gw.yaml') - + # Build dependency list gw_deps = [] if start_mysql: gw_deps.append('mysql') # Add compilation dependency for Unkey services gw_deps.append('unkey-compile') - + k8s_resource( 'gw', port_forwards='6060:6060', @@ -199,7 +198,7 @@ if start_gw: # Ctrl service (1 replica) if start_ctrl: print("Setting up Ctrl service...") - + docker_build_with_restart( 'unkey-ctrl:latest', '.', @@ -211,16 +210,16 @@ if start_ctrl: ], ignore=['./cmd/api', './cmd/gw', './apps/api', './apps/gw'] ) - + k8s_yaml('k8s/manifests/ctrl.yaml') - + # Build dependency list ctrl_deps = [] if start_mysql: ctrl_deps.append('mysql') if start_s3: ctrl_deps.append('s3') # Add compilation dependency for Unkey services ctrl_deps.append('unkey-compile') - + k8s_resource( 'ctrl', port_forwards='7091:7091', @@ -244,12 +243,12 @@ if start_dashboard: ] ) k8s_yaml('k8s/manifests/dashboard.yaml') - + # Build dependency list dashboard_deps = [] if start_planetscale: dashboard_deps.append('planetscale') if start_clickhouse: dashboard_deps.append('clickhouse') - + k8s_resource( 'dashboard', port_forwards='3000:3000', @@ -263,7 +262,7 @@ if start_dashboard: active_services = [] if redis_started: active_services.append('redis') if start_mysql: active_services.append('mysql') -if start_clickhouse: active_services.append('clickhouse') +if start_clickhouse: active_services.append('clickhouse') if start_s3: active_services.append('s3') if start_planetscale: active_services.append('planetscale') if start_observability: active_services.extend(['prometheus', 'otel-collector']) @@ -282,10 +281,10 @@ Web UI: http://localhost:10350 Services available via Tilt port forwards: Dashboard: http://localhost:3000 API: http://localhost:7070 -Gateway: http://localhost:6060 +Gateway: http://localhost:6060 Ctrl: http://localhost:7091 Prometheus: http://localhost:9090 -S3 Console: http://localhost:9000 +S3 Console: http://localhost:9000 ClickHouse: http://localhost:8123 Tips: @@ -299,4 +298,4 @@ Development workflow: - View logs in unified Tilt UI - Use ingress for service access -""" % ', '.join(active_services)) \ No newline at end of file +""" % ', '.join(active_services)) diff --git a/go/k8s/manifests/api.yaml b/go/k8s/manifests/api.yaml index 6f263e9c5c..5040458273 100644 --- a/go/k8s/manifests/api.yaml +++ b/go/k8s/manifests/api.yaml @@ -17,82 +17,87 @@ spec: app: api spec: containers: - - name: api - image: unkey-api:latest - imagePullPolicy: Never # Use local images - ports: - - containerPort: 7070 - env: - # Server Configuration - - name: UNKEY_HTTP_PORT - value: "7070" - - name: UNKEY_LOGS_COLOR - value: "true" - - name: UNKEY_TEST_MODE - value: "false" - # Instance Identification - - name: UNKEY_PLATFORM - value: "kubernetes" - - name: UNKEY_IMAGE - value: "unkey:latest" - - name: UNKEY_REGION - value: "local" - - name: UNKEY_INSTANCE_ID - value: "api-dev" - # Database Configuration - - name: UNKEY_DATABASE_PRIMARY - value: "unkey:password@tcp(mysql:3306)/unkey?parseTime=true&interpolateParams=true" - # Caching and Storage - - name: UNKEY_REDIS_URL - value: "redis://redis:6379" - - name: UNKEY_CLICKHOUSE_URL - value: "clickhouse://unkey:password@clickhouse:9000?secure=false&skip_verify=true" - # Observability - DISABLED for development - - name: UNKEY_OTEL - value: "false" - - name: UNKEY_PROMETHEUS_PORT - value: "0" - # Vault Configuration - - name: UNKEY_VAULT_MASTER_KEYS - value: "Ch9rZWtfMmdqMFBJdVhac1NSa0ZhNE5mOWlLSnBHenFPENTt7an5MRogENt9Si6wms4pQ2XIvqNSIgNpaBenJmXgcInhu6Nfv2U=" - - name: UNKEY_VAULT_S3_URL - value: "http://s3:9000" - - name: UNKEY_VAULT_S3_BUCKET - value: "vault" - - name: UNKEY_VAULT_S3_ACCESS_KEY_ID - value: "minio_root_user" - - name: UNKEY_VAULT_S3_ACCESS_KEY_SECRET - value: "minio_root_password" - # ClickHouse Proxy Service Configuration - - name: UNKEY_CHPROXY_AUTH_TOKEN - value: "chproxy-test-token-123" - # Request Body Configuration - - name: UNKEY_MAX_REQUEST_BODY_SIZE - value: "10485760" - command: ["/unkey", "run", "api"] - readinessProbe: - httpGet: - path: /v2/liveness - port: 7070 - initialDelaySeconds: 10 - periodSeconds: 5 - livenessProbe: - httpGet: - path: /v2/liveness - port: 7070 - initialDelaySeconds: 30 - periodSeconds: 10 - resources: - requests: - memory: "256Mi" - cpu: "200m" - limits: - memory: "1Gi" - cpu: "1000m" + - name: api + image: unkey-api:latest + imagePullPolicy: Never # Use local images + ports: + - containerPort: 7070 + env: + # Server Configuration + - name: UNKEY_HTTP_PORT + value: "7070" + - name: UNKEY_LOGS_COLOR + value: "true" + - name: UNKEY_TEST_MODE + value: "false" + # Instance Identification + - name: UNKEY_PLATFORM + value: "kubernetes" + - name: UNKEY_IMAGE + value: "unkey:latest" + - name: UNKEY_REGION + value: "local" + - name: UNKEY_INSTANCE_ID + value: "api-dev" + # Database Configuration + - name: UNKEY_DATABASE_PRIMARY + value: "unkey:password@tcp(mysql:3306)/unkey?parseTime=true&interpolateParams=true" + # Caching and Storage + - name: UNKEY_REDIS_URL + value: "redis://redis:6379" + - name: UNKEY_CLICKHOUSE_URL + value: "clickhouse://unkey:password@clickhouse:9000?secure=false&skip_verify=true" + # Observability - DISABLED for development + - name: UNKEY_OTEL + value: "false" + - name: UNKEY_PROMETHEUS_PORT + value: "0" + # Vault Configuration + - name: UNKEY_VAULT_MASTER_KEYS + value: "Ch9rZWtfMmdqMFBJdVhac1NSa0ZhNE5mOWlLSnBHenFPENTt7an5MRogENt9Si6wms4pQ2XIvqNSIgNpaBenJmXgcInhu6Nfv2U=" + - name: UNKEY_VAULT_S3_URL + value: "http://s3:9000" + - name: UNKEY_VAULT_S3_BUCKET + value: "vault" + - name: UNKEY_VAULT_S3_ACCESS_KEY_ID + value: "minio_root_user" + - name: UNKEY_VAULT_S3_ACCESS_KEY_SECRET + value: "minio_root_password" + # ClickHouse Proxy Service Configuration + - name: UNKEY_CHPROXY_AUTH_TOKEN + value: "chproxy-test-token-123" + # Request Body Configuration + - name: UNKEY_MAX_REQUEST_BODY_SIZE + value: "10485760" + command: ["/unkey", "run", "api"] + readinessProbe: + httpGet: + path: /v2/liveness + port: 7070 + initialDelaySeconds: 10 + periodSeconds: 5 + livenessProbe: + httpGet: + path: /v2/liveness + port: 7070 + initialDelaySeconds: 30 + periodSeconds: 10 + resources: + requests: + memory: "256Mi" + cpu: "200m" + limits: + memory: "1Gi" + cpu: "1000m" initContainers: - - name: wait-for-dependencies - image: busybox:1.36 - command: ['sh', '-c', 'until nc -z mysql 3306 && nc -z clickhouse 8123; do echo waiting for dependencies; sleep 2; done;'] + - name: wait-for-dependencies + image: busybox:1.36 + command: + [ + "sh", + "-c", + "until nc -z mysql 3306 && nc -z clickhouse 8123; do echo waiting for dependencies; sleep 2; done;", + ] --- apiVersion: v1 @@ -106,10 +111,10 @@ spec: selector: app: api ports: - - name: http - port: 7070 - targetPort: 7070 - protocol: TCP + - name: http + port: 7070 + targetPort: 7070 + protocol: TCP type: NodePort --- @@ -125,8 +130,8 @@ spec: selector: app: api ports: - - name: http - port: 80 - targetPort: 7070 - protocol: TCP - type: LoadBalancer \ No newline at end of file + - name: http + port: 80 + targetPort: 7070 + protocol: TCP + type: LoadBalancer diff --git a/go/k8s/manifests/clickhouse.yaml b/go/k8s/manifests/clickhouse.yaml index 4a06255023..a2ac35b39a 100644 --- a/go/k8s/manifests/clickhouse.yaml +++ b/go/k8s/manifests/clickhouse.yaml @@ -30,46 +30,46 @@ spec: app: clickhouse spec: containers: - - name: clickhouse - image: clickhouse/clickhouse-server:24.3 - ports: - - containerPort: 8123 - name: http - - containerPort: 9000 - name: native - env: - - name: CLICKHOUSE_DB - value: "unkey" - - name: CLICKHOUSE_USER - value: "unkey" - - name: CLICKHOUSE_PASSWORD - value: "password" - volumeMounts: - - name: clickhouse-storage - mountPath: /var/lib/clickhouse - readinessProbe: - httpGet: - path: /ping - port: 8123 - initialDelaySeconds: 30 - periodSeconds: 10 - livenessProbe: - httpGet: - path: /ping - port: 8123 - initialDelaySeconds: 60 - periodSeconds: 30 - resources: - requests: - memory: "256Mi" - cpu: "200m" - limits: - memory: "1Gi" - cpu: "1000m" + - name: clickhouse + image: clickhouse/clickhouse-server:24.3 + ports: + - containerPort: 8123 + name: http + - containerPort: 9000 + name: native + env: + - name: CLICKHOUSE_DB + value: "unkey" + - name: CLICKHOUSE_USER + value: "unkey" + - name: CLICKHOUSE_PASSWORD + value: "password" + volumeMounts: + - name: clickhouse-storage + mountPath: /var/lib/clickhouse + readinessProbe: + httpGet: + path: /ping + port: 8123 + initialDelaySeconds: 30 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /ping + port: 8123 + initialDelaySeconds: 60 + periodSeconds: 30 + resources: + requests: + memory: "256Mi" + cpu: "200m" + limits: + memory: "1Gi" + cpu: "1000m" volumes: - - name: clickhouse-storage - persistentVolumeClaim: - claimName: clickhouse-pvc + - name: clickhouse-storage + persistentVolumeClaim: + claimName: clickhouse-pvc --- apiVersion: v1 @@ -83,12 +83,12 @@ spec: selector: app: clickhouse ports: - - name: http - port: 8123 - targetPort: 8123 - protocol: TCP - - name: native - port: 9000 - targetPort: 9000 - protocol: TCP - type: ClusterIP \ No newline at end of file + - name: http + port: 8123 + targetPort: 8123 + protocol: TCP + - name: native + port: 9000 + targetPort: 9000 + protocol: TCP + type: ClusterIP diff --git a/go/k8s/manifests/ctrl.yaml b/go/k8s/manifests/ctrl.yaml index 8f34d1af9e..ae687df4d3 100644 --- a/go/k8s/manifests/ctrl.yaml +++ b/go/k8s/manifests/ctrl.yaml @@ -18,71 +18,76 @@ spec: spec: serviceAccountName: ctrl-service-account containers: - - name: ctrl - image: unkey-ctrl:latest - imagePullPolicy: Never # Use local images - ports: - - containerPort: 7091 - env: - # Server Configuration - - name: UNKEY_HTTP_PORT - value: "7091" - - name: UNKEY_LOGS_COLOR - value: "true" - # Instance Identification - - name: UNKEY_PLATFORM - value: "kubernetes" - - name: UNKEY_IMAGE - value: "unkey:latest" - - name: UNKEY_REGION - value: "local" - - name: UNKEY_INSTANCE_ID - value: "ctrl-dev" - # Database Configuration - - name: UNKEY_DATABASE_PRIMARY - value: "unkey:password@tcp(mysql:3306)/unkey?parseTime=true&interpolateParams=true" - - name: UNKEY_DATABASE_PARTITION - value: "unkey:password@tcp(mysql:3306)/partition_001?parseTime=true&interpolateParams=true" - - name: UNKEY_DATABASE_HYDRA - value: "unkey:password@tcp(mysql:3306)/hydra?parseTime=true&interpolateParams=true" - # Observability - DISABLED for development - - name: UNKEY_OTEL - value: "false" - # Control Plane Specific - - name: UNKEY_AUTH_TOKEN - value: "dev-auth-token" - - name: UNKEY_METALD_ADDRESS - value: "http://metald-placeholder:8080" - - name: UNKEY_SPIFFE_SOCKET_PATH - value: "/var/lib/spire/agent/agent.sock" - # Vault Configuration (required) - - name: UNKEY_VAULT_MASTER_KEYS - value: "Ch9rZWtfMmdqMFBJdVhac1NSa0ZhNE5mOWlLSnBHenFPENTt7an5MRogENt9Si6wms4pQ2XIvqNSIgNpaBenJmXgcInhu6Nfv2U=" - - name: UNKEY_VAULT_S3_URL - value: "http://s3:9000" - - name: UNKEY_VAULT_S3_BUCKET - value: "vault" - - name: UNKEY_VAULT_S3_ACCESS_KEY_ID - value: "minio_root_user" - - name: UNKEY_VAULT_S3_ACCESS_KEY_SECRET - value: "minio_root_password" - # Additional Configuration - - name: UNKEY_ACME_ENABLED - value: "false" - - name: UNKEY_METALD_FALLBACK - value: "k8s" - command: ["/unkey", "run", "ctrl"] - resources: - requests: - memory: "128Mi" - cpu: "100m" - limits: - memory: "512Mi" - cpu: "500m" + - name: ctrl + image: unkey-ctrl:latest + imagePullPolicy: Never # Use local images + ports: + - containerPort: 7091 + env: + # Server Configuration + - name: UNKEY_HTTP_PORT + value: "7091" + - name: UNKEY_LOGS_COLOR + value: "true" + # Instance Identification + - name: UNKEY_PLATFORM + value: "kubernetes" + - name: UNKEY_IMAGE + value: "unkey:latest" + - name: UNKEY_REGION + value: "local" + - name: UNKEY_INSTANCE_ID + value: "ctrl-dev" + # Database Configuration + - name: UNKEY_DATABASE_PRIMARY + value: "unkey:password@tcp(mysql:3306)/unkey?parseTime=true&interpolateParams=true" + - name: UNKEY_DATABASE_PARTITION + value: "unkey:password@tcp(mysql:3306)/partition_001?parseTime=true&interpolateParams=true" + - name: UNKEY_DATABASE_HYDRA + value: "unkey:password@tcp(mysql:3306)/hydra?parseTime=true&interpolateParams=true" + # Observability - DISABLED for development + - name: UNKEY_OTEL + value: "false" + # Control Plane Specific + - name: UNKEY_AUTH_TOKEN + value: "dev-auth-token" + - name: UNKEY_METALD_ADDRESS + value: "http://metald-placeholder:8080" + - name: UNKEY_SPIFFE_SOCKET_PATH + value: "/var/lib/spire/agent/agent.sock" + # Vault Configuration (required) + - name: UNKEY_VAULT_MASTER_KEYS + value: "Ch9rZWtfMmdqMFBJdVhac1NSa0ZhNE5mOWlLSnBHenFPENTt7an5MRogENt9Si6wms4pQ2XIvqNSIgNpaBenJmXgcInhu6Nfv2U=" + - name: UNKEY_VAULT_S3_URL + value: "http://s3:9000" + - name: UNKEY_VAULT_S3_BUCKET + value: "vault" + - name: UNKEY_VAULT_S3_ACCESS_KEY_ID + value: "minio_root_user" + - name: UNKEY_VAULT_S3_ACCESS_KEY_SECRET + value: "minio_root_password" + # Additional Configuration + - name: UNKEY_ACME_ENABLED + value: "false" + - name: UNKEY_METALD_FALLBACK + value: "k8s" + command: ["/unkey", "run", "ctrl"] + resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "512Mi" + cpu: "500m" initContainers: - - name: wait-for-dependencies - image: busybox:1.36 - command: ['sh', '-c', 'until nc -z mysql 3306 && nc -z s3 9000; do echo waiting for dependencies; sleep 2; done;'] + - name: wait-for-dependencies + image: busybox:1.36 + command: + [ + "sh", + "-c", + "until nc -z mysql 3306 && nc -z s3 9000; do echo waiting for dependencies; sleep 2; done;", + ] --- apiVersion: v1 @@ -96,9 +101,8 @@ spec: selector: app: ctrl ports: - - name: http - port: 7091 - targetPort: 7091 - protocol: TCP + - name: http + port: 7091 + targetPort: 7091 + protocol: TCP type: NodePort - diff --git a/go/k8s/manifests/dashboard.yaml b/go/k8s/manifests/dashboard.yaml index 317962de83..de63ae7b95 100644 --- a/go/k8s/manifests/dashboard.yaml +++ b/go/k8s/manifests/dashboard.yaml @@ -17,56 +17,56 @@ spec: app: dashboard spec: initContainers: - - name: wait-for-dependencies - image: busybox:1.36 - command: - - sh - - -c - - | - until nc -z planetscale 3900; do - echo waiting for planetscale - sleep 2 - done + - name: wait-for-dependencies + image: busybox:1.36 + command: + - sh + - -c + - | + until nc -z planetscale 3900; do + echo waiting for planetscale + sleep 2 + done containers: - - name: dashboard - image: unkey/dashboard:latest - imagePullPolicy: Never - ports: - - containerPort: 3000 - env: - # Database configuration - - name: DATABASE_HOST - value: "planetscale:3900" - # ClickHouse configuration - - name: CLICKHOUSE_URL - value: "http://unkey:password@clickhouse:8123" - # Environment - - name: NODE_ENV - value: "production" - # Instance identification - - name: UNKEY_PLATFORM - value: "kubernetes" - - name: UNKEY_REGION - value: "local" - readinessProbe: - httpGet: - path: / - port: 3000 - initialDelaySeconds: 10 - periodSeconds: 5 - livenessProbe: - httpGet: - path: / - port: 3000 - initialDelaySeconds: 30 - periodSeconds: 10 - resources: - requests: - memory: "256Mi" - cpu: "200m" - limits: - memory: "512Mi" - cpu: "500m" + - name: dashboard + image: unkey/dashboard:latest + imagePullPolicy: Never + ports: + - containerPort: 3000 + env: + # Database configuration + - name: DATABASE_HOST + value: "planetscale:3900" + # ClickHouse configuration + - name: CLICKHOUSE_URL + value: "http://unkey:password@clickhouse:8123" + # Environment + - name: NODE_ENV + value: "production" + # Instance identification + - name: UNKEY_PLATFORM + value: "kubernetes" + - name: UNKEY_REGION + value: "local" + readinessProbe: + httpGet: + path: / + port: 3000 + initialDelaySeconds: 10 + periodSeconds: 5 + livenessProbe: + httpGet: + path: / + port: 3000 + initialDelaySeconds: 30 + periodSeconds: 10 + resources: + requests: + memory: "256Mi" + cpu: "200m" + limits: + memory: "512Mi" + cpu: "500m" --- apiVersion: v1 @@ -80,8 +80,8 @@ spec: selector: app: dashboard ports: - - name: http - port: 3000 - targetPort: 3000 - protocol: TCP - type: NodePort \ No newline at end of file + - name: http + port: 3000 + targetPort: 3000 + protocol: TCP + type: NodePort diff --git a/go/k8s/manifests/gw.yaml b/go/k8s/manifests/gw.yaml index 912258a395..c67c8e657e 100644 --- a/go/k8s/manifests/gw.yaml +++ b/go/k8s/manifests/gw.yaml @@ -17,87 +17,92 @@ spec: app: gw spec: containers: - - name: gw - image: unkey-gw:latest - imagePullPolicy: Never # Use local images - ports: - - containerPort: 6060 - - containerPort: 6443 - env: - # Server Configuration - - name: UNKEY_HTTP_PORT - value: "6060" - - name: UNKEY_HTTPS_PORT - value: "6443" - - name: UNKEY_TLS_ENABLED - value: "false" - # Instance Identification - - name: UNKEY_PLATFORM - value: "kubernetes" - - name: UNKEY_IMAGE - value: "unkey:latest" - - name: UNKEY_REGION - value: "local" - - name: UNKEY_GATEWAY_ID - value: "gw-dev" - - name: UNKEY_DEFAULT_CERT_DOMAIN - value: "" - - name: UNKEY_MAIN_DOMAIN - value: "" - - name: UNKEY_CTRL_ADDR - value: "ctrl:7091" - # Database Configuration - Partitioned (for gateway operations) - - name: UNKEY_DATABASE_PRIMARY - value: "unkey:password@tcp(mysql:3306)/partition_001?parseTime=true&interpolateParams=true" - # Database Configuration - Keys Service - - name: UNKEY_KEYS_DATABASE_PRIMARY - value: "unkey:password@tcp(mysql:3306)/unkey?parseTime=true&interpolateParams=true" - # ClickHouse Configuration - - name: UNKEY_CLICKHOUSE_URL - value: "clickhouse://unkey:password@clickhouse:9000?secure=false&skip_verify=true" - # Redis Configuration - - name: UNKEY_REDIS_URL - value: "redis://redis:6379" - # Observability - DISABLED for development - - name: UNKEY_OTEL - value: "false" - - name: UNKEY_PROMETHEUS_PORT - value: "0" - # Vault Configuration - - name: UNKEY_VAULT_MASTER_KEYS - value: "Ch9rZWtfMmdqMFBJdVhac1NSa0ZhNE5mOWlLSnBHenFPENTt7an5MRogENt9Si6wms4pQ2XIvqNSIgNpaBenJmXgcInhu6Nfv2U=" - - name: UNKEY_VAULT_S3_URL - value: "http://s3:9000" - - name: UNKEY_VAULT_S3_BUCKET - value: "vault" - - name: UNKEY_VAULT_S3_ACCESS_KEY_ID - value: "minio_root_user" - - name: UNKEY_VAULT_S3_ACCESS_KEY_SECRET - value: "minio_root_password" - command: ["/unkey", "run", "gw"] - readinessProbe: - httpGet: - path: /unkey/_internal/liveness - port: 6060 - initialDelaySeconds: 10 - periodSeconds: 5 - livenessProbe: - httpGet: - path: /unkey/_internal/liveness - port: 6060 - initialDelaySeconds: 30 - periodSeconds: 10 - resources: - requests: - memory: "128Mi" - cpu: "100m" - limits: - memory: "512Mi" - cpu: "500m" + - name: gw + image: unkey-gw:latest + imagePullPolicy: Never # Use local images + ports: + - containerPort: 8080 + - containerPort: 8443 + env: + # Server Configuration + - name: UNKEY_HTTP_PORT + value: "8080" + - name: UNKEY_HTTPS_PORT + value: "8443" + - name: UNKEY_TLS_ENABLED + value: "false" + # Instance Identification + - name: UNKEY_PLATFORM + value: "kubernetes" + - name: UNKEY_IMAGE + value: "unkey:latest" + - name: UNKEY_REGION + value: "local" + - name: UNKEY_GATEWAY_ID + value: "gw-dev" + - name: UNKEY_DEFAULT_CERT_DOMAIN + value: "" + - name: UNKEY_MAIN_DOMAIN + value: "" + - name: UNKEY_CTRL_ADDR + value: "ctrl:7091" + # Database Configuration - Partitioned (for gateway operations) + - name: UNKEY_DATABASE_PRIMARY + value: "unkey:password@tcp(mysql:3306)/partition_001?parseTime=true&interpolateParams=true" + # Database Configuration - Keys Service + - name: UNKEY_KEYS_DATABASE_PRIMARY + value: "unkey:password@tcp(mysql:3306)/unkey?parseTime=true&interpolateParams=true" + # ClickHouse Configuration + - name: UNKEY_CLICKHOUSE_URL + value: "clickhouse://unkey:password@clickhouse:9000?secure=false&skip_verify=true" + # Redis Configuration + - name: UNKEY_REDIS_URL + value: "redis://redis:6379" + # Observability - DISABLED for development + - name: UNKEY_OTEL + value: "false" + - name: UNKEY_PROMETHEUS_PORT + value: "0" + # Vault Configuration + - name: UNKEY_VAULT_MASTER_KEYS + value: "Ch9rZWtfMmdqMFBJdVhac1NSa0ZhNE5mOWlLSnBHenFPENTt7an5MRogENt9Si6wms4pQ2XIvqNSIgNpaBenJmXgcInhu6Nfv2U=" + - name: UNKEY_VAULT_S3_URL + value: "http://s3:9000" + - name: UNKEY_VAULT_S3_BUCKET + value: "vault" + - name: UNKEY_VAULT_S3_ACCESS_KEY_ID + value: "minio_root_user" + - name: UNKEY_VAULT_S3_ACCESS_KEY_SECRET + value: "minio_root_password" + command: ["/unkey", "run", "gw"] + readinessProbe: + httpGet: + path: /unkey/_internal/liveness + port: 8080 + initialDelaySeconds: 10 + periodSeconds: 5 + livenessProbe: + httpGet: + path: /unkey/_internal/liveness + port: 8080 + initialDelaySeconds: 30 + periodSeconds: 10 + resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "512Mi" + cpu: "500m" initContainers: - - name: wait-for-dependencies - image: busybox:1.36 - command: ['sh', '-c', 'until nc -z mysql 3306; do echo waiting for dependencies; sleep 2; done;'] + - name: wait-for-dependencies + image: busybox:1.36 + command: + [ + "sh", + "-c", + "until nc -z mysql 3306; do echo waiting for dependencies; sleep 2; done;", + ] --- apiVersion: v1 @@ -111,35 +116,12 @@ spec: selector: app: gw ports: - - name: http - port: 6060 - targetPort: 6060 - protocol: TCP - - name: https - port: 6443 - targetPort: 6443 - protocol: TCP - type: NodePort - ---- -apiVersion: v1 -kind: Service -metadata: - name: gw-lb - namespace: unkey - labels: - app: gw - service: loadbalancer -spec: - selector: - app: gw - ports: - - name: http - port: 80 - targetPort: 6060 - protocol: TCP - - name: https - port: 443 - targetPort: 6443 - protocol: TCP - type: LoadBalancer \ No newline at end of file + - name: http + port: 80 + targetPort: 8080 + protocol: TCP + - name: https + port: 443 + targetPort: 8443 + protocol: TCP + type: LoadBalancer diff --git a/go/k8s/manifests/ingress.yaml b/go/k8s/manifests/ingress.yaml deleted file mode 100644 index a495d63b67..0000000000 --- a/go/k8s/manifests/ingress.yaml +++ /dev/null @@ -1,2 +0,0 @@ -# OrbStack handles LoadBalancer services automatically -# Services are accessible at .k8s.orb.local \ No newline at end of file diff --git a/go/k8s/manifests/mysql.yaml b/go/k8s/manifests/mysql.yaml index e3c2e18d51..95d4e05cad 100644 --- a/go/k8s/manifests/mysql.yaml +++ b/go/k8s/manifests/mysql.yaml @@ -30,47 +30,47 @@ spec: app: mysql spec: containers: - - name: mysql - image: unkey/mysql:latest - imagePullPolicy: Never - ports: - - containerPort: 3306 - env: - - name: MYSQL_ROOT_PASSWORD - value: "root" - - name: MYSQL_DATABASE - value: "unkey" - - name: MYSQL_USER - value: "unkey" - - name: MYSQL_PASSWORD - value: "password" - args: - - "--max_connections=1000" - volumeMounts: - - name: mysql-storage - mountPath: /var/lib/mysql - readinessProbe: - exec: - command: - - mysqladmin - - ping - - -h - - localhost - initialDelaySeconds: 30 - periodSeconds: 10 - livenessProbe: - exec: - command: - - mysqladmin - - ping - - -h - - localhost - initialDelaySeconds: 60 - periodSeconds: 30 + - name: mysql + image: unkey/mysql:latest + imagePullPolicy: Never + ports: + - containerPort: 3306 + env: + - name: MYSQL_ROOT_PASSWORD + value: "root" + - name: MYSQL_DATABASE + value: "unkey" + - name: MYSQL_USER + value: "unkey" + - name: MYSQL_PASSWORD + value: "password" + args: + - "--max_connections=1000" + volumeMounts: + - name: mysql-storage + mountPath: /var/lib/mysql + readinessProbe: + exec: + command: + - mysqladmin + - ping + - -h + - localhost + initialDelaySeconds: 30 + periodSeconds: 10 + livenessProbe: + exec: + command: + - mysqladmin + - ping + - -h + - localhost + initialDelaySeconds: 60 + periodSeconds: 30 volumes: - - name: mysql-storage - persistentVolumeClaim: - claimName: mysql-pvc + - name: mysql-storage + persistentVolumeClaim: + claimName: mysql-pvc --- apiVersion: v1 @@ -84,7 +84,7 @@ spec: selector: app: mysql ports: - - port: 3306 - targetPort: 3306 - protocol: TCP - type: ClusterIP \ No newline at end of file + - port: 3306 + targetPort: 3306 + protocol: TCP + type: ClusterIP diff --git a/go/k8s/manifests/namespace.yaml b/go/k8s/manifests/namespace.yaml index 192d856504..cdc6e8d819 100644 --- a/go/k8s/manifests/namespace.yaml +++ b/go/k8s/manifests/namespace.yaml @@ -4,4 +4,4 @@ metadata: name: unkey labels: app.kubernetes.io/name: unkey - app.kubernetes.io/instance: dev \ No newline at end of file + app.kubernetes.io/instance: dev diff --git a/go/k8s/manifests/observability.yaml b/go/k8s/manifests/observability.yaml index d3fcd0039d..982610bde9 100644 --- a/go/k8s/manifests/observability.yaml +++ b/go/k8s/manifests/observability.yaml @@ -34,31 +34,31 @@ spec: app: prometheus spec: containers: - - name: prometheus - image: prom/prometheus:v2.45.0 - ports: - - containerPort: 9090 - args: - - '--config.file=/etc/prometheus/prometheus.yml' - - '--storage.tsdb.path=/prometheus/' - - '--web.console.libraries=/etc/prometheus/console_libraries' - - '--web.console.templates=/etc/prometheus/consoles' - - '--storage.tsdb.retention.time=200h' - - '--web.enable-lifecycle' - volumeMounts: - - name: prometheus-config - mountPath: /etc/prometheus/ - resources: - requests: - memory: "128Mi" - cpu: "100m" - limits: - memory: "512Mi" - cpu: "500m" + - name: prometheus + image: prom/prometheus:v2.45.0 + ports: + - containerPort: 9090 + args: + - "--config.file=/etc/prometheus/prometheus.yml" + - "--storage.tsdb.path=/prometheus/" + - "--web.console.libraries=/etc/prometheus/console_libraries" + - "--web.console.templates=/etc/prometheus/consoles" + - "--storage.tsdb.retention.time=200h" + - "--web.enable-lifecycle" + volumeMounts: + - name: prometheus-config + mountPath: /etc/prometheus/ + resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "512Mi" + cpu: "500m" volumes: - - name: prometheus-config - configMap: - name: prometheus-config + - name: prometheus-config + configMap: + name: prometheus-config --- apiVersion: v1 @@ -72,9 +72,9 @@ spec: selector: app: prometheus ports: - - port: 9090 - targetPort: 9090 - protocol: TCP + - port: 9090 + targetPort: 9090 + protocol: TCP type: NodePort --- @@ -96,32 +96,32 @@ spec: app: otel-collector spec: containers: - - name: otel-collector - image: otel/opentelemetry-collector-contrib:0.92.0 - ports: - - containerPort: 4317 # OTLP gRPC - - containerPort: 4318 # OTLP HTTP - - containerPort: 8889 # Prometheus metrics - command: - - /otelcol-contrib - - --config=/etc/otelcol-contrib/otel-collector.yml - env: - - name: GOMEMLIMIT - value: "160MiB" - volumeMounts: - - name: otel-collector-config - mountPath: /etc/otelcol-contrib/ - resources: - requests: - memory: "128Mi" - cpu: "100m" - limits: - memory: "256Mi" - cpu: "200m" + - name: otel-collector + image: otel/opentelemetry-collector-contrib:0.92.0 + ports: + - containerPort: 4317 # OTLP gRPC + - containerPort: 4318 # OTLP HTTP + - containerPort: 8889 # Prometheus metrics + command: + - /otelcol-contrib + - --config=/etc/otelcol-contrib/otel-collector.yml + env: + - name: GOMEMLIMIT + value: "160MiB" + volumeMounts: + - name: otel-collector-config + mountPath: /etc/otelcol-contrib/ + resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "256Mi" + cpu: "200m" volumes: - - name: otel-collector-config - configMap: - name: otel-collector-config + - name: otel-collector-config + configMap: + name: otel-collector-config --- apiVersion: v1 @@ -182,16 +182,16 @@ spec: selector: app: otel-collector ports: - - name: otlp-grpc - port: 4317 - targetPort: 4317 - protocol: TCP - - name: otlp-http - port: 4318 - targetPort: 4318 - protocol: TCP - - name: prometheus - port: 8889 - targetPort: 8889 - protocol: TCP - type: NodePort \ No newline at end of file + - name: otlp-grpc + port: 4317 + targetPort: 4317 + protocol: TCP + - name: otlp-http + port: 4318 + targetPort: 4318 + protocol: TCP + - name: prometheus + port: 8889 + targetPort: 8889 + protocol: TCP + type: NodePort diff --git a/go/k8s/manifests/planetscale.yaml b/go/k8s/manifests/planetscale.yaml index a3551d3eb1..7643259410 100644 --- a/go/k8s/manifests/planetscale.yaml +++ b/go/k8s/manifests/planetscale.yaml @@ -17,37 +17,44 @@ spec: app: planetscale spec: containers: - - name: planetscale-http - image: ghcr.io/mattrobenolt/ps-http-sim:v0.0.12 - ports: - - containerPort: 3900 - args: ["-listen-port=3900", "-mysql-dbname=unkey", "-mysql-addr=mysql", "-mysql-max-rows=100000", "-mysql-idle-timeout=1s"] - env: - - name: DATABASE_HOST - value: "mysql" - - name: DATABASE_NAME - value: "unkey" - - name: DATABASE_USERNAME - value: "unkey" - - name: DATABASE_PASSWORD - value: "password" - readinessProbe: - tcpSocket: - port: 3900 - initialDelaySeconds: 10 - periodSeconds: 5 - livenessProbe: - tcpSocket: - port: 3900 - initialDelaySeconds: 30 - periodSeconds: 10 - resources: - requests: - memory: "64Mi" - cpu: "50m" - limits: - memory: "256Mi" - cpu: "200m" + - name: planetscale-http + image: ghcr.io/mattrobenolt/ps-http-sim:v0.0.12 + ports: + - containerPort: 3900 + args: + [ + "-listen-port=3900", + "-mysql-dbname=unkey", + "-mysql-addr=mysql", + "-mysql-max-rows=100000", + "-mysql-idle-timeout=1s", + ] + env: + - name: DATABASE_HOST + value: "mysql" + - name: DATABASE_NAME + value: "unkey" + - name: DATABASE_USERNAME + value: "unkey" + - name: DATABASE_PASSWORD + value: "password" + readinessProbe: + tcpSocket: + port: 3900 + initialDelaySeconds: 10 + periodSeconds: 5 + livenessProbe: + tcpSocket: + port: 3900 + initialDelaySeconds: 30 + periodSeconds: 10 + resources: + requests: + memory: "64Mi" + cpu: "50m" + limits: + memory: "256Mi" + cpu: "200m" --- apiVersion: v1 @@ -61,7 +68,7 @@ spec: selector: app: planetscale ports: - - port: 3900 - targetPort: 3900 - protocol: TCP - type: ClusterIP \ No newline at end of file + - port: 3900 + targetPort: 3900 + protocol: TCP + type: ClusterIP diff --git a/go/k8s/manifests/rbac.yaml b/go/k8s/manifests/rbac.yaml index 074232ace7..6b2294ede5 100644 --- a/go/k8s/manifests/rbac.yaml +++ b/go/k8s/manifests/rbac.yaml @@ -15,15 +15,15 @@ metadata: labels: app: ctrl rules: -- apiGroups: ["apps"] - resources: ["deployments"] - verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] -- apiGroups: [""] - resources: ["pods", "services"] - verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] -- apiGroups: [""] - resources: ["configmaps", "secrets"] - verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: ["apps"] + resources: ["deployments"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: [""] + resources: ["pods", "services"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: [""] + resources: ["configmaps", "secrets"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] --- apiVersion: rbac.authorization.k8s.io/v1 @@ -33,10 +33,10 @@ metadata: labels: app: ctrl subjects: -- kind: ServiceAccount - name: ctrl-service-account - namespace: unkey + - kind: ServiceAccount + name: ctrl-service-account + namespace: unkey roleRef: kind: ClusterRole name: ctrl-deployment-manager - apiGroup: rbac.authorization.k8s.io \ No newline at end of file + apiGroup: rbac.authorization.k8s.io diff --git a/go/k8s/manifests/redis.yaml b/go/k8s/manifests/redis.yaml index f843f09911..2672a85218 100644 --- a/go/k8s/manifests/redis.yaml +++ b/go/k8s/manifests/redis.yaml @@ -17,31 +17,31 @@ spec: app: redis spec: containers: - - name: redis - image: redis:8.0 - ports: - - containerPort: 6379 - readinessProbe: - exec: - command: - - redis-cli - - ping - initialDelaySeconds: 5 - periodSeconds: 5 - livenessProbe: - exec: - command: - - redis-cli - - ping - initialDelaySeconds: 10 - periodSeconds: 30 - resources: - requests: - memory: "64Mi" - cpu: "50m" - limits: - memory: "256Mi" - cpu: "200m" + - name: redis + image: redis:8.0 + ports: + - containerPort: 6379 + readinessProbe: + exec: + command: + - redis-cli + - ping + initialDelaySeconds: 5 + periodSeconds: 5 + livenessProbe: + exec: + command: + - redis-cli + - ping + initialDelaySeconds: 10 + periodSeconds: 30 + resources: + requests: + memory: "64Mi" + cpu: "50m" + limits: + memory: "256Mi" + cpu: "200m" --- apiVersion: v1 @@ -55,7 +55,7 @@ spec: selector: app: redis ports: - - port: 6379 - targetPort: 6379 - protocol: TCP - type: ClusterIP \ No newline at end of file + - port: 6379 + targetPort: 6379 + protocol: TCP + type: ClusterIP diff --git a/go/k8s/manifests/s3.yaml b/go/k8s/manifests/s3.yaml index e0aeead721..e283fa7b4a 100644 --- a/go/k8s/manifests/s3.yaml +++ b/go/k8s/manifests/s3.yaml @@ -30,50 +30,50 @@ spec: app: s3 spec: containers: - - name: minio - image: bitnami/minio:2025.7.23 - ports: - - containerPort: 9000 - name: api - - containerPort: 9001 - name: console - env: - - name: MINIO_ROOT_USER - value: "minio_root_user" - - name: MINIO_ROOT_PASSWORD - value: "minio_root_password" - - name: MINIO_API_PORT_NUMBER - value: "9000" - - name: MINIO_CONSOLE_PORT_NUMBER - value: "9001" - - name: MINIO_DEFAULT_BUCKETS - value: "vault" - volumeMounts: - - name: s3-storage - mountPath: /data - readinessProbe: - httpGet: - path: /minio/health/ready - port: 9000 - initialDelaySeconds: 30 - periodSeconds: 10 - livenessProbe: - httpGet: - path: /minio/health/live - port: 9000 - initialDelaySeconds: 60 - periodSeconds: 30 - resources: - requests: - memory: "128Mi" - cpu: "100m" - limits: - memory: "512Mi" - cpu: "500m" + - name: minio + image: bitnami/minio:2025.7.23 + ports: + - containerPort: 9000 + name: api + - containerPort: 9001 + name: console + env: + - name: MINIO_ROOT_USER + value: "minio_root_user" + - name: MINIO_ROOT_PASSWORD + value: "minio_root_password" + - name: MINIO_API_PORT_NUMBER + value: "9000" + - name: MINIO_CONSOLE_PORT_NUMBER + value: "9001" + - name: MINIO_DEFAULT_BUCKETS + value: "vault" + volumeMounts: + - name: s3-storage + mountPath: /data + readinessProbe: + httpGet: + path: /minio/health/ready + port: 9000 + initialDelaySeconds: 30 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /minio/health/live + port: 9000 + initialDelaySeconds: 60 + periodSeconds: 30 + resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "512Mi" + cpu: "500m" volumes: - - name: s3-storage - persistentVolumeClaim: - claimName: s3-pvc + - name: s3-storage + persistentVolumeClaim: + claimName: s3-pvc --- apiVersion: v1 @@ -87,10 +87,10 @@ spec: selector: app: s3 ports: - - name: api - port: 9000 - targetPort: 9000 - protocol: TCP + - name: api + port: 9000 + targetPort: 9000 + protocol: TCP type: NodePort --- @@ -106,8 +106,8 @@ spec: selector: app: s3 ports: - - name: console - port: 9001 - targetPort: 9001 - protocol: TCP - type: NodePort \ No newline at end of file + - name: console + port: 9001 + targetPort: 9001 + protocol: TCP + type: NodePort From 2d2f54d4477db99ea2d2cd9d9dd62282ec8fc3a2 Mon Sep 17 00:00:00 2001 From: Flo Date: Thu, 11 Sep 2025 10:53:37 +0200 Subject: [PATCH 4/9] expose ports to localhost by default --- go/k8s/manifests/api.yaml | 6 +++--- go/k8s/manifests/clickhouse.yaml | 6 +++--- go/k8s/manifests/ctrl.yaml | 8 ++++---- go/k8s/manifests/dashboard.yaml | 2 +- go/k8s/manifests/gw.yaml | 4 ++-- go/k8s/manifests/mysql.yaml | 2 +- go/k8s/manifests/planetscale.yaml | 2 +- go/k8s/manifests/redis.yaml | 2 +- go/k8s/manifests/s3.yaml | 8 ++++---- 9 files changed, 20 insertions(+), 20 deletions(-) diff --git a/go/k8s/manifests/api.yaml b/go/k8s/manifests/api.yaml index 5040458273..e5577ee2e4 100644 --- a/go/k8s/manifests/api.yaml +++ b/go/k8s/manifests/api.yaml @@ -46,7 +46,7 @@ spec: - name: UNKEY_REDIS_URL value: "redis://redis:6379" - name: UNKEY_CLICKHOUSE_URL - value: "clickhouse://unkey:password@clickhouse:9000?secure=false&skip_verify=true" + value: "clickhouse://default:password@clickhouse:9000?secure=false&skip_verify=true" # Observability - DISABLED for development - name: UNKEY_OTEL value: "false" @@ -56,7 +56,7 @@ spec: - name: UNKEY_VAULT_MASTER_KEYS value: "Ch9rZWtfMmdqMFBJdVhac1NSa0ZhNE5mOWlLSnBHenFPENTt7an5MRogENt9Si6wms4pQ2XIvqNSIgNpaBenJmXgcInhu6Nfv2U=" - name: UNKEY_VAULT_S3_URL - value: "http://s3:9000" + value: "http://s3:3902" - name: UNKEY_VAULT_S3_BUCKET value: "vault" - name: UNKEY_VAULT_S3_ACCESS_KEY_ID @@ -115,7 +115,7 @@ spec: port: 7070 targetPort: 7070 protocol: TCP - type: NodePort + type: LoadBalancer --- apiVersion: v1 diff --git a/go/k8s/manifests/clickhouse.yaml b/go/k8s/manifests/clickhouse.yaml index a2ac35b39a..043f9e4332 100644 --- a/go/k8s/manifests/clickhouse.yaml +++ b/go/k8s/manifests/clickhouse.yaml @@ -39,9 +39,9 @@ spec: name: native env: - name: CLICKHOUSE_DB - value: "unkey" + value: "default" - name: CLICKHOUSE_USER - value: "unkey" + value: "default" - name: CLICKHOUSE_PASSWORD value: "password" volumeMounts: @@ -91,4 +91,4 @@ spec: port: 9000 targetPort: 9000 protocol: TCP - type: ClusterIP + type: LoadBalancer diff --git a/go/k8s/manifests/ctrl.yaml b/go/k8s/manifests/ctrl.yaml index ae687df4d3..bcc8a1b954 100644 --- a/go/k8s/manifests/ctrl.yaml +++ b/go/k8s/manifests/ctrl.yaml @@ -59,7 +59,7 @@ spec: - name: UNKEY_VAULT_MASTER_KEYS value: "Ch9rZWtfMmdqMFBJdVhac1NSa0ZhNE5mOWlLSnBHenFPENTt7an5MRogENt9Si6wms4pQ2XIvqNSIgNpaBenJmXgcInhu6Nfv2U=" - name: UNKEY_VAULT_S3_URL - value: "http://s3:9000" + value: "http://s3:3902" - name: UNKEY_VAULT_S3_BUCKET value: "vault" - name: UNKEY_VAULT_S3_ACCESS_KEY_ID @@ -69,7 +69,7 @@ spec: # Additional Configuration - name: UNKEY_ACME_ENABLED value: "false" - - name: UNKEY_METALD_FALLBACK + - name: UNKEY_METALD_BACKEND value: "k8s" command: ["/unkey", "run", "ctrl"] resources: @@ -86,7 +86,7 @@ spec: [ "sh", "-c", - "until nc -z mysql 3306 && nc -z s3 9000; do echo waiting for dependencies; sleep 2; done;", + "until nc -z mysql 3306 && nc -z s3 3902; do echo waiting for dependencies; sleep 2; done;", ] --- @@ -105,4 +105,4 @@ spec: port: 7091 targetPort: 7091 protocol: TCP - type: NodePort + type: LoadBalancer diff --git a/go/k8s/manifests/dashboard.yaml b/go/k8s/manifests/dashboard.yaml index de63ae7b95..c3c0e595ca 100644 --- a/go/k8s/manifests/dashboard.yaml +++ b/go/k8s/manifests/dashboard.yaml @@ -84,4 +84,4 @@ spec: port: 3000 targetPort: 3000 protocol: TCP - type: NodePort + type: LoadBalancer diff --git a/go/k8s/manifests/gw.yaml b/go/k8s/manifests/gw.yaml index c67c8e657e..744c2bc6d8 100644 --- a/go/k8s/manifests/gw.yaml +++ b/go/k8s/manifests/gw.yaml @@ -54,7 +54,7 @@ spec: value: "unkey:password@tcp(mysql:3306)/unkey?parseTime=true&interpolateParams=true" # ClickHouse Configuration - name: UNKEY_CLICKHOUSE_URL - value: "clickhouse://unkey:password@clickhouse:9000?secure=false&skip_verify=true" + value: "clickhouse://default:password@clickhouse:9000?secure=false&skip_verify=true" # Redis Configuration - name: UNKEY_REDIS_URL value: "redis://redis:6379" @@ -67,7 +67,7 @@ spec: - name: UNKEY_VAULT_MASTER_KEYS value: "Ch9rZWtfMmdqMFBJdVhac1NSa0ZhNE5mOWlLSnBHenFPENTt7an5MRogENt9Si6wms4pQ2XIvqNSIgNpaBenJmXgcInhu6Nfv2U=" - name: UNKEY_VAULT_S3_URL - value: "http://s3:9000" + value: "http://s3:3902" - name: UNKEY_VAULT_S3_BUCKET value: "vault" - name: UNKEY_VAULT_S3_ACCESS_KEY_ID diff --git a/go/k8s/manifests/mysql.yaml b/go/k8s/manifests/mysql.yaml index 95d4e05cad..78e2e615d2 100644 --- a/go/k8s/manifests/mysql.yaml +++ b/go/k8s/manifests/mysql.yaml @@ -87,4 +87,4 @@ spec: - port: 3306 targetPort: 3306 protocol: TCP - type: ClusterIP + type: LoadBalancer diff --git a/go/k8s/manifests/planetscale.yaml b/go/k8s/manifests/planetscale.yaml index 7643259410..219e2995ec 100644 --- a/go/k8s/manifests/planetscale.yaml +++ b/go/k8s/manifests/planetscale.yaml @@ -71,4 +71,4 @@ spec: - port: 3900 targetPort: 3900 protocol: TCP - type: ClusterIP + type: LoadBalancer diff --git a/go/k8s/manifests/redis.yaml b/go/k8s/manifests/redis.yaml index 2672a85218..2508991552 100644 --- a/go/k8s/manifests/redis.yaml +++ b/go/k8s/manifests/redis.yaml @@ -58,4 +58,4 @@ spec: - port: 6379 targetPort: 6379 protocol: TCP - type: ClusterIP + type: LoadBalancer diff --git a/go/k8s/manifests/s3.yaml b/go/k8s/manifests/s3.yaml index e283fa7b4a..7b651c7352 100644 --- a/go/k8s/manifests/s3.yaml +++ b/go/k8s/manifests/s3.yaml @@ -88,10 +88,10 @@ spec: app: s3 ports: - name: api - port: 9000 + port: 3902 targetPort: 9000 protocol: TCP - type: NodePort + type: LoadBalancer --- apiVersion: v1 @@ -107,7 +107,7 @@ spec: app: s3 ports: - name: console - port: 9001 + port: 3903 targetPort: 9001 protocol: TCP - type: NodePort + type: LoadBalancer From 8ce300d042acba91f138947c66444c87cc950e7d Mon Sep 17 00:00:00 2001 From: Flo Date: Thu, 11 Sep 2025 11:16:33 +0200 Subject: [PATCH 5/9] update makefile --- go/Makefile | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/go/Makefile b/go/Makefile index f5e20de3f1..adef4e1b7b 100644 --- a/go/Makefile +++ b/go/Makefile @@ -111,6 +111,11 @@ k8s-up: k8s-check ## Deploy all services to current Kubernetes cluster @echo "Check NodePort assignments: make k8s-ports" @make k8s-status +k8s-stop: ## Stop all pods (scale deployments to 0) + @echo "Stopping all pods..." + @kubectl scale deployment --all --replicas=0 -n unkey + @echo "All pods stopped" + k8s-down: ## Delete all services from current Kubernetes cluster @echo "Deleting all services..." @kubectl delete namespace unkey --ignore-not-found=true @@ -118,16 +123,6 @@ k8s-down: ## Delete all services from current Kubernetes cluster k8s-reset: k8s-down k8s-up ## Reset the entire Kubernetes environment -k8s-status: ## Show status of k8s services - @echo "Cluster Status:" - @kubectl get nodes -o wide 2>/dev/null || echo "ERROR: Cluster not running" - @echo "\nServices Status:" - @kubectl get pods,svc -n unkey 2>/dev/null || echo "ERROR: Services not deployed" - -k8s-ports: ## Show NodePort assignments for direct localhost access - @echo "Services accessible via NodePort on localhost:" - @kubectl get svc -n unkey -o custom-columns="NAME:.metadata.name,TYPE:.spec.type,PORT(S):.spec.ports[*].port,NODEPORT:.spec.ports[*].nodePort" | grep NodePort - # Helper function to deploy a service define deploy-service @echo "Starting $(2)..." From 3b0de47dce7732212980cb1aacbe1096023db4c1 Mon Sep 17 00:00:00 2001 From: Flo Date: Thu, 11 Sep 2025 20:27:48 +0200 Subject: [PATCH 6/9] remove resources --- go/Makefile | 2 ++ go/Tiltfile | 10 ++++++ go/k8s/manifests/api.yaml | 7 ---- go/k8s/manifests/clickhouse.yaml | 51 ++++++++++++++++++----------- go/k8s/manifests/ctrl.yaml | 7 ---- go/k8s/manifests/dashboard.yaml | 7 ---- go/k8s/manifests/gw.yaml | 7 ---- go/k8s/manifests/mysql.yaml | 2 +- go/k8s/manifests/observability.yaml | 14 -------- go/k8s/manifests/planetscale.yaml | 7 ---- go/k8s/manifests/redis.yaml | 7 ---- go/k8s/manifests/s3.yaml | 9 +---- 12 files changed, 45 insertions(+), 85 deletions(-) diff --git a/go/Makefile b/go/Makefile index f3cd667f6f..4749db6658 100644 --- a/go/Makefile +++ b/go/Makefile @@ -81,6 +81,7 @@ k8s-check: ## Check if kubectl is available and cluster is running k8s-up: k8s-check ## Deploy all services to current Kubernetes cluster @echo "Building Docker images..." @docker build -t unkey/mysql:latest -f ../deployment/Dockerfile.mysql ../ + @docker build -t unkey/clickhouse:latest -f ../deployment/Dockerfile.clickhouse ../ @docker build -t unkey:latest . @echo "Creating namespace..." @kubectl apply -f k8s/manifests/namespace.yaml @@ -128,6 +129,7 @@ start-mysql: k8s-check ## Deploy only MySQL $(call deploy-service,mysql,MySQL,unkey/mysql:latest,-f ../deployment/Dockerfile.mysql ../) start-clickhouse: k8s-check ## Deploy only ClickHouse + @docker build -t unkey/clickhouse:latest -f ../deployment/Dockerfile.clickhouse ../ $(call deploy-service,clickhouse,ClickHouse) start-redis: k8s-check ## Deploy only Redis diff --git a/go/Tiltfile b/go/Tiltfile index 7ef4fa545c..92020b9473 100644 --- a/go/Tiltfile +++ b/go/Tiltfile @@ -69,6 +69,16 @@ if start_mysql: # ClickHouse service if start_clickhouse: print("Setting up ClickHouse...") + # Only build if image doesn't exist + if not local('docker images -q unkey/clickhouse:latest 2>/dev/null', quiet=True): + print("Building ClickHouse image...") + docker_build( + 'unkey/clickhouse:latest', + '../', # Build context at repo root + dockerfile='../deployment/Dockerfile.clickhouse' + ) + else: + print("Using existing ClickHouse image") k8s_yaml('k8s/manifests/clickhouse.yaml') k8s_resource( 'clickhouse', diff --git a/go/k8s/manifests/api.yaml b/go/k8s/manifests/api.yaml index e5577ee2e4..6d7f945516 100644 --- a/go/k8s/manifests/api.yaml +++ b/go/k8s/manifests/api.yaml @@ -82,13 +82,6 @@ spec: port: 7070 initialDelaySeconds: 30 periodSeconds: 10 - resources: - requests: - memory: "256Mi" - cpu: "200m" - limits: - memory: "1Gi" - cpu: "1000m" initContainers: - name: wait-for-dependencies image: busybox:1.36 diff --git a/go/k8s/manifests/clickhouse.yaml b/go/k8s/manifests/clickhouse.yaml index 043f9e4332..fd9fa919d5 100644 --- a/go/k8s/manifests/clickhouse.yaml +++ b/go/k8s/manifests/clickhouse.yaml @@ -9,7 +9,7 @@ spec: - ReadWriteOnce resources: requests: - storage: 2Gi + storage: 100Gi --- apiVersion: apps/v1 @@ -31,41 +31,52 @@ spec: spec: containers: - name: clickhouse - image: clickhouse/clickhouse-server:24.3 + image: unkey/clickhouse:latest ports: - containerPort: 8123 name: http - containerPort: 9000 name: native env: - - name: CLICKHOUSE_DB + - name: CLICKHOUSE_ADMIN_USER value: "default" - - name: CLICKHOUSE_USER - value: "default" - - name: CLICKHOUSE_PASSWORD + - name: CLICKHOUSE_ADMIN_PASSWORD value: "password" + - name: ALLOW_EMPTY_PASSWORD + value: "no" volumeMounts: - name: clickhouse-storage - mountPath: /var/lib/clickhouse + mountPath: /bitnami/clickhouse readinessProbe: - httpGet: - path: /ping - port: 8123 + exec: + command: + - clickhouse-client + - --host + - localhost + - --user + - default + - --password + - password + - --query + - "SELECT 1" initialDelaySeconds: 30 periodSeconds: 10 + timeoutSeconds: 5 livenessProbe: - httpGet: - path: /ping - port: 8123 + exec: + command: + - clickhouse-client + - --host + - localhost + - --user + - default + - --password + - password + - --query + - "SELECT 1" initialDelaySeconds: 60 periodSeconds: 30 - resources: - requests: - memory: "256Mi" - cpu: "200m" - limits: - memory: "1Gi" - cpu: "1000m" + timeoutSeconds: 5 volumes: - name: clickhouse-storage persistentVolumeClaim: diff --git a/go/k8s/manifests/ctrl.yaml b/go/k8s/manifests/ctrl.yaml index bcc8a1b954..f0d9a6e5e9 100644 --- a/go/k8s/manifests/ctrl.yaml +++ b/go/k8s/manifests/ctrl.yaml @@ -72,13 +72,6 @@ spec: - name: UNKEY_METALD_BACKEND value: "k8s" command: ["/unkey", "run", "ctrl"] - resources: - requests: - memory: "128Mi" - cpu: "100m" - limits: - memory: "512Mi" - cpu: "500m" initContainers: - name: wait-for-dependencies image: busybox:1.36 diff --git a/go/k8s/manifests/dashboard.yaml b/go/k8s/manifests/dashboard.yaml index c3c0e595ca..ec59e7a046 100644 --- a/go/k8s/manifests/dashboard.yaml +++ b/go/k8s/manifests/dashboard.yaml @@ -60,13 +60,6 @@ spec: port: 3000 initialDelaySeconds: 30 periodSeconds: 10 - resources: - requests: - memory: "256Mi" - cpu: "200m" - limits: - memory: "512Mi" - cpu: "500m" --- apiVersion: v1 diff --git a/go/k8s/manifests/gw.yaml b/go/k8s/manifests/gw.yaml index 744c2bc6d8..d118f20963 100644 --- a/go/k8s/manifests/gw.yaml +++ b/go/k8s/manifests/gw.yaml @@ -87,13 +87,6 @@ spec: port: 8080 initialDelaySeconds: 30 periodSeconds: 10 - resources: - requests: - memory: "128Mi" - cpu: "100m" - limits: - memory: "512Mi" - cpu: "500m" initContainers: - name: wait-for-dependencies image: busybox:1.36 diff --git a/go/k8s/manifests/mysql.yaml b/go/k8s/manifests/mysql.yaml index 78e2e615d2..25366af105 100644 --- a/go/k8s/manifests/mysql.yaml +++ b/go/k8s/manifests/mysql.yaml @@ -9,7 +9,7 @@ spec: - ReadWriteOnce resources: requests: - storage: 1Gi + storage: 100Gi --- apiVersion: apps/v1 diff --git a/go/k8s/manifests/observability.yaml b/go/k8s/manifests/observability.yaml index 982610bde9..ec7034a488 100644 --- a/go/k8s/manifests/observability.yaml +++ b/go/k8s/manifests/observability.yaml @@ -48,13 +48,6 @@ spec: volumeMounts: - name: prometheus-config mountPath: /etc/prometheus/ - resources: - requests: - memory: "128Mi" - cpu: "100m" - limits: - memory: "512Mi" - cpu: "500m" volumes: - name: prometheus-config configMap: @@ -111,13 +104,6 @@ spec: volumeMounts: - name: otel-collector-config mountPath: /etc/otelcol-contrib/ - resources: - requests: - memory: "128Mi" - cpu: "100m" - limits: - memory: "256Mi" - cpu: "200m" volumes: - name: otel-collector-config configMap: diff --git a/go/k8s/manifests/planetscale.yaml b/go/k8s/manifests/planetscale.yaml index 219e2995ec..612ae65134 100644 --- a/go/k8s/manifests/planetscale.yaml +++ b/go/k8s/manifests/planetscale.yaml @@ -48,13 +48,6 @@ spec: port: 3900 initialDelaySeconds: 30 periodSeconds: 10 - resources: - requests: - memory: "64Mi" - cpu: "50m" - limits: - memory: "256Mi" - cpu: "200m" --- apiVersion: v1 diff --git a/go/k8s/manifests/redis.yaml b/go/k8s/manifests/redis.yaml index 2508991552..523d806e18 100644 --- a/go/k8s/manifests/redis.yaml +++ b/go/k8s/manifests/redis.yaml @@ -35,13 +35,6 @@ spec: - ping initialDelaySeconds: 10 periodSeconds: 30 - resources: - requests: - memory: "64Mi" - cpu: "50m" - limits: - memory: "256Mi" - cpu: "200m" --- apiVersion: v1 diff --git a/go/k8s/manifests/s3.yaml b/go/k8s/manifests/s3.yaml index 7b651c7352..96314dad6c 100644 --- a/go/k8s/manifests/s3.yaml +++ b/go/k8s/manifests/s3.yaml @@ -9,7 +9,7 @@ spec: - ReadWriteOnce resources: requests: - storage: 5Gi + storage: 50Gi --- apiVersion: apps/v1 @@ -63,13 +63,6 @@ spec: port: 9000 initialDelaySeconds: 60 periodSeconds: 30 - resources: - requests: - memory: "128Mi" - cpu: "100m" - limits: - memory: "512Mi" - cpu: "500m" volumes: - name: s3-storage persistentVolumeClaim: From ef51b2d2bdc4521eb0f17767e813aa587412b0ee Mon Sep 17 00:00:00 2001 From: Flo Date: Mon, 15 Sep 2025 10:57:42 +0200 Subject: [PATCH 7/9] update rbac --- go/k8s/manifests/rbac.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/go/k8s/manifests/rbac.yaml b/go/k8s/manifests/rbac.yaml index 6b2294ede5..9523d7a690 100644 --- a/go/k8s/manifests/rbac.yaml +++ b/go/k8s/manifests/rbac.yaml @@ -24,6 +24,9 @@ rules: - apiGroups: [""] resources: ["configmaps", "secrets"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] --- apiVersion: rbac.authorization.k8s.io/v1 From 7f94eb69eb02037afd6f90a1b2707e07de694a00 Mon Sep 17 00:00:00 2001 From: Flo Date: Mon, 15 Sep 2025 12:40:44 +0200 Subject: [PATCH 8/9] sure --- go/Tiltfile | 2 +- go/k8s/manifests/gw.yaml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go/Tiltfile b/go/Tiltfile index 92020b9473..63d3fc0ccf 100644 --- a/go/Tiltfile +++ b/go/Tiltfile @@ -198,7 +198,7 @@ if start_gw: k8s_resource( 'gw', - port_forwards='6060:6060', + port_forwards=['8080:8080', '8443:8443'], resource_deps=gw_deps, labels=['unkey'], auto_init=True, diff --git a/go/k8s/manifests/gw.yaml b/go/k8s/manifests/gw.yaml index d118f20963..1abd03dfea 100644 --- a/go/k8s/manifests/gw.yaml +++ b/go/k8s/manifests/gw.yaml @@ -110,11 +110,11 @@ spec: app: gw ports: - name: http - port: 80 + port: 8080 targetPort: 8080 protocol: TCP - name: https - port: 443 + port: 8443 targetPort: 8443 protocol: TCP - type: LoadBalancer + type: ClusterIP From 548f067deae002249302099384544820dd4272c6 Mon Sep 17 00:00:00 2001 From: Flo Date: Mon, 15 Sep 2025 14:22:11 +0200 Subject: [PATCH 9/9] cleanup md --- go/K8S_DEVELOPMENT.md | 273 +++++------------------------------------- 1 file changed, 27 insertions(+), 246 deletions(-) diff --git a/go/K8S_DEVELOPMENT.md b/go/K8S_DEVELOPMENT.md index e6d81c8eb6..e0251f0c25 100644 --- a/go/K8S_DEVELOPMENT.md +++ b/go/K8S_DEVELOPMENT.md @@ -1,284 +1,65 @@ # Kubernetes Development Setup -This guide shows how to run Unkey services locally using Docker Desktop or OrbStack Kubernetes instead of Docker Compose. +Run Unkey services locally using Kubernetes instead of Docker Compose. ## Prerequisites -Install the required tools: - -- **Docker Desktop with Kubernetes enabled** OR **OrbStack with Kubernetes enabled** -- **kubectl**: https://kubernetes.io/docs/tasks/tools/ - -Or check if already installed: +- Docker Desktop with Kubernetes enabled OR OrbStack with Kubernetes enabled +- kubectl +Check requirements: ```bash make k8s-check ``` -Optional for enhanced development experience: - -- **Tilt**: https://docs.tilt.dev/install.html - ## Quick Start -### Option 1: Full Environment (Recommended) - +Start everything: ```bash make k8s-up ``` -This will: - -- Use your current Kubernetes cluster (Docker Desktop/OrbStack) -- Build and deploy all services (MySQL, ClickHouse, S3, Observability, Unkey) -- Wait for all services to be ready -- Show connection info - -### Option 2: Individual Services - -```bash -# Start only MySQL -make start-mysql - -# Start only ctrl (requires cluster to exist) -make start-ctrl - -# Start all services individually -make start-all -``` - -### Option 3: Enhanced Development with Tilt - +Start with hot reloading (requires Tilt): ```bash make dev ``` -If Tilt is installed, this provides: - -- Hot reloading for Go code changes -- Unified log viewing in web UI (http://localhost:10350) -- Resource management dashboard -- Automatic rebuilds on file changes -- Selective service startup - -## Accessing Services - -### Via NodePort (OrbStack) - -Services are accessible directly on localhost via NodePort. Check actual port assignments with: +## Individual Services ```bash -make k8s-ports +make start-mysql +make start-clickhouse +make start-redis +make start-s3 +make start-api +make start-gw +make start-ctrl ``` -This will show the randomly assigned NodePorts, e.g.: - -- **Dashboard**: http://localhost:3000 -- **API**: http://localhost:32002 -- **Gateway**: http://localhost:32003 -- **Ctrl**: http://localhost:32004 -- **Prometheus**: http://localhost:32005 -- **S3 Console**: http://localhost:32006 - -### Alternative: LoadBalancer domains - -OrbStack also supports LoadBalancer services with automatic `*.k8s.orb.local` domains, but NodePort is simpler for development. - -### Inside the cluster - -- **MySQL**: `mysql.unkey.svc.cluster.local:3306` -- **ClickHouse**: `clickhouse.unkey.svc.cluster.local:8123` -- **S3**: `s3.unkey.svc.cluster.local:9000` -- **Prometheus**: `prometheus.unkey.svc.cluster.local:9090` -- **OTEL Collector**: `otel-collector.unkey.svc.cluster.local:4317` - -## Development Workflow - -### Making Code Changes - -#### With Tilt (Hot Reloading) - -1. Start Tilt: `make dev` -2. Edit Go files -3. Changes automatically rebuild and restart services -4. View logs in Tilt UI - -#### Without Tilt (Manual) - -1. Make code changes -2. Rebuild and redeploy: - ```bash - make start-ctrl # Rebuilds and redeploys ctrl - ``` - -### Managing the Environment +## Management ```bash -# Check status -make k8s-status - -# Reset everything (delete and recreate) -make k8s-reset - # Stop everything make k8s-down -``` - -### Debugging -```bash -# View logs -kubectl logs -n unkey -l app=ctrl -f -kubectl logs -n unkey -l app=mysql -f - -# Get shell access -kubectl exec -n unkey -it deployment/ctrl -- /bin/sh -kubectl exec -n unkey -it deployment/mysql -- /bin/bash +# Reset environment +make k8s-reset -# Check service health +# View services kubectl get pods -n unkey -kubectl describe pod -n unkey +kubectl get services -n unkey ``` -## Configuration - -### Environment Variables - -The ctrl service uses these key environment variables: - -- `UNKEY_DATABASE_PRIMARY`: Connection to partition database -- `UNKEY_DATABASE_HYDRA`: Connection to hydra database -- `UNKEY_HTTP_PORT`: Service port (8084) -- `UNKEY_PLATFORM`: Set to "kubernetes" - -### Database Setup - -MySQL is automatically configured with: - -- `unkey` database for main data -- `hydra` database for OAuth flows -- `partition_001` database for partitioned data -- User `unkey` with password `password` - -All schemas are automatically applied on startup. - -## Customization - -### Selective Services with Tilt - -```bash -# Start only database services -tilt up mysql,clickhouse,planetscale - -# Start only storage services -tilt up s3 - -# Start only observability -tilt up observability - -# Start only Unkey services -tilt up ctrl -``` - -### Custom Resource Limits - -Edit `k8s/manifests/ctrl.yaml` to adjust: - -```yaml -resources: - requests: - memory: "128Mi" - cpu: "100m" - limits: - memory: "512Mi" - cpu: "500m" -``` - -### Persistent Data - -MySQL data persists between restarts via PersistentVolumeClaim. To reset: - -```bash -kubectl delete pvc -n unkey mysql-pvc -make start-mysql -``` - -## Troubleshooting - -### Common Issues - -#### Cluster Not Available +## Tilt (Optional) +Start specific services: ```bash -# Make sure Kubernetes is enabled in Docker Desktop/OrbStack -# Check current context -kubectl config current-context - -# Switch context if needed -kubectl config use-context docker-desktop # or orbstack -``` - -#### Services Not Ready - -```bash -# Check events -kubectl get events -n unkey --sort-by='.lastTimestamp' - -# Check pod logs -kubectl logs -n unkey -l app=mysql -kubectl logs -n unkey -l app=ctrl +tilt up -- --services=mysql --services=clickhouse +tilt up -- --services=api --services=gw --services=ctrl +tilt up -- --services=all ``` -#### Port Conflicts - -If ports 8084 or 3306 are in use: - +Stop Tilt: ```bash -# Check what's using the port -lsof -i :8084 -lsof -i :3306 - -# Kill other services or change ports in k8s manifests -``` - -#### Image Build Issues - -```bash -# Manual build -docker build -t unkey/mysql:latest -f ../deployment/Dockerfile.mysql ../ -docker build -t unkey:latest . -``` - -### Performance Tips - -- Use `make dev` for fastest development cycle -- Keep cluster running between sessions (don't run `k8s-down`) -- Use selective service startup when working on specific components - -## vs Docker Compose - -| Feature | Kubernetes | Docker Compose | -| --------------------- | -------------------------------- | ---------------------- | -| **Prod Similarity** | ✅ Real Kubernetes | ❌ Different from prod | -| **Resource Usage** | ✅ Native containers | ✅ Native containers | -| **Setup Complexity** | ⚠️ Enable K8s in Docker/OrbStack | ✅ Simple setup | -| **Service Discovery** | ✅ K8s native DNS | ⚠️ Docker networks | -| **Scaling** | ✅ Easy horizontal scaling | ❌ Limited scaling | -| **Hot Reloading** | ✅ With Tilt | ⚠️ Manual restarts | -| **Debugging** | ✅ Rich tooling | ✅ Simple logs | -| **Architecture** | ✅ Clean separation | ✅ Clean separation | - -Choose Kubernetes for true prod-like development, Docker Compose for quick iterations. - -## Available Services - -All services can be started individually or together: - -- `mysql` - MySQL database with schemas -- `clickhouse` - ClickHouse analytics database -- `s3` - MinIO object storage -- `planetscale` - PlanetScale HTTP database proxy -- `observability` - Prometheus + OTEL Collector -- `ctrl` - Main Unkey service (ctrl/api/gw) - -Use `make k8s-up` to start everything or selectively with Tilt. +tilt down +``` \ No newline at end of file