diff --git a/deployment/docker-compose.yaml b/deployment/docker-compose.yaml index 401adf5ff2..28f93da2e0 100644 --- a/deployment/docker-compose.yaml +++ b/deployment/docker-compose.yaml @@ -193,6 +193,8 @@ services: - "7091:7091" depends_on: - mysql + - metald-aio + - otel environment: # Database configuration - use existing mysql service UNKEY_DATABASE_PRIMARY: "unkey:password@tcp(mysql:3306)/unkey?parseTime=true" @@ -200,7 +202,7 @@ services: # Control plane configuration UNKEY_HTTP_PORT: "7091" - UNKEY_METALD_ADDRESS: "http://localhost:8080" + UNKEY_METALD_ADDRESS: "http://metald-aio:8080" # Override the entrypoint to run ctrl command @@ -256,8 +258,46 @@ services: UNKEY_WORKSPACE_ID: "ws_local_root" UNKEY_API_ID: "api_local_root_keys" + # Unkey Deploy Services - All-in-one development container with all 4 services + metald-aio: + build: + context: ../go + dockerfile: deploy/Dockerfile.dev + platform: linux/amd64 + container_name: metald-aio + hostname: metald-aio + privileged: true # Required for systemd + volumes: + # Mount Docker socket for metald Docker backend + - /var/run/docker.sock:/var/run/docker.sock + # Persistent storage for development + - metald-aio-data:/opt + # Systemd requires these mounts + - /sys/fs/cgroup:/sys/fs/cgroup:ro + ports: + # Only expose metald port (agent already uses 8080) + - "8090:8080" # metald + depends_on: + - otel + environment: + # Development environment + - UNKEY_ENVIRONMENT=development + - UNKEY_LOG_LEVEL=debug + + # Docker backend configuration + - UNKEY_METALD_BACKEND=docker + - UNKEY_METALD_DOCKER_HOST=unix:///var/run/docker.sock + + # Service discovery (internal container ports) + - UNKEY_METALD_BILLING_ENDPOINT=http://localhost:8081 + - UNKEY_METALD_ASSETMANAGER_ENDPOINT=http://localhost:8083 + - UNKEY_ASSETMANAGERD_BUILDERD_ENDPOINT=http://localhost:8082 + - UNKEY_BUILDERD_ASSETMANAGER_ENDPOINT=http://localhost:8083 + + volumes: mysql: clickhouse: clickhouse-keeper: s3: + metald-aio-data: diff --git a/go/deploy/Dockerfile.dev b/go/deploy/Dockerfile.dev new file mode 100644 index 0000000000..b5deff6e8d --- /dev/null +++ b/go/deploy/Dockerfile.dev @@ -0,0 +1,225 @@ +# Dockerfile.dev - Development environment for all Unkey deploy services +# Based on LOCAL_DEPLOYMENT_GUIDE.md for maximum production parity + +# Build stage - compile all services +FROM fedora:42 AS builder + +# Install development tools (following LOCAL_DEPLOYMENT_GUIDE.md) +RUN dnf install -y dnf-plugins-core && \ + dnf group install -y development-tools && \ + dnf install -y git make golang curl wget iptables-legacy && \ + dnf clean all + + +# Set up Go environment +ENV GOPATH=/go +ENV PATH=$PATH:/go/bin:/usr/local/go/bin + +# Copy source code +COPY . /src/go +WORKDIR /src/go + +# Protobuf files are already generated in go/proto/ - no need to generate them again + +# Build all services directly using go build (protobufs already generated) +# Go will download dependencies as needed during build +WORKDIR /src/go/deploy/assetmanagerd +RUN go build -o assetmanagerd ./cmd/assetmanagerd + +WORKDIR /src/go/deploy/billaged +RUN go build -o billaged ./cmd/billaged + +WORKDIR /src/go/deploy/builderd +RUN go build -o builderd ./cmd/builderd + +WORKDIR /src/go/deploy/metald +RUN go build -o metald ./cmd/metald + +# Runtime stage - Fedora with systemd +FROM fedora:42 + +# Install runtime dependencies +RUN dnf update -y && \ + dnf install -y \ + systemd \ + systemd-devel \ + iptables-legacy \ + curl \ + wget \ + procps-ng \ + util-linux \ + && \ + dnf clean all + +# Install Docker CLI for metald Docker backend +RUN dnf install -y dnf-plugins-core && \ + dnf config-manager addrepo --from-repofile=https://download.docker.com/linux/fedora/docker-ce.repo && \ + dnf install -y docker-ce-cli && \ + dnf clean all + +# Create billaged user (following systemd service requirements) +RUN useradd -r -s /bin/false billaged + +# Create required directories (following LOCAL_DEPLOYMENT_GUIDE.md structure) +RUN mkdir -p /opt/assetmanagerd/{cache,data} && \ + mkdir -p /opt/billaged && \ + mkdir -p /opt/builderd/{scratch,rootfs,workspace,data} && \ + mkdir -p /opt/metald/{sockets,logs,assets} && \ + mkdir -p /srv/jailer && \ + mkdir -p /var/log/{assetmanagerd,billaged,builderd,metald} && \ + mkdir -p /opt/vm-assets + +# Set ownership for service directories +RUN chown -R billaged:billaged /opt/billaged /var/log/billaged + +# Copy built binaries from builder stage +COPY --from=builder /src/go/deploy/assetmanagerd/assetmanagerd /usr/local/bin/ +COPY --from=builder /src/go/deploy/billaged/billaged /usr/local/bin/ +COPY --from=builder /src/go/deploy/builderd/builderd /usr/local/bin/ +COPY --from=builder /src/go/deploy/metald/metald /usr/local/bin/ + + +# Make binaries executable +RUN chmod +x /usr/local/bin/assetmanagerd* /usr/local/bin/billaged* /usr/local/bin/builderd* /usr/local/bin/metald* + +# Copy systemd service files for container environment +COPY deploy/docker/systemd/ /etc/systemd/system/ + +# Create development environment file +RUN cat > /etc/default/unkey-deploy << 'EOF' +# Development environment variables for all services +# TLS disabled for development +UNKEY_ASSETMANAGERD_TLS_MODE=disabled +UNKEY_BILLAGED_TLS_MODE=disabled +UNKEY_BUILDERD_TLS_MODE=disabled +UNKEY_METALD_TLS_MODE=disabled + +# Configure Docker backend for metald +UNKEY_METALD_BACKEND=docker + +# Service endpoints (HTTP for development) +UNKEY_METALD_BILLING_ENDPOINT=http://localhost:8081 +UNKEY_METALD_ASSETMANAGER_ENDPOINT=http://localhost:8083 +UNKEY_ASSETMANAGERD_BUILDERD_ENDPOINT=http://localhost:8082 +UNKEY_BUILDERD_ASSETMANAGER_ENDPOINT=http://localhost:8083 + +# Service configuration +UNKEY_ASSETMANAGERD_PORT=8083 +UNKEY_ASSETMANAGERD_ADDRESS=0.0.0.0 +UNKEY_BILLAGED_PORT=8081 +UNKEY_BILLAGED_ADDRESS=0.0.0.0 +UNKEY_BUILDERD_PORT=8082 +UNKEY_BUILDERD_ADDRESS=0.0.0.0 +UNKEY_METALD_PORT=8080 +UNKEY_METALD_ADDRESS=0.0.0.0 + +# Storage configuration +UNKEY_ASSETMANAGERD_STORAGE_BACKEND=local +UNKEY_ASSETMANAGERD_LOCAL_STORAGE_PATH=/opt/builderd/rootfs +UNKEY_ASSETMANAGERD_DATABASE_PATH=/opt/assetmanagerd/assets.db +UNKEY_ASSETMANAGERD_CACHE_DIR=/opt/assetmanagerd/cache + +# Build configuration +UNKEY_BUILDERD_SCRATCH_DIR=/opt/builderd/scratch +UNKEY_BUILDERD_ROOTFS_OUTPUT_DIR=/opt/builderd/rootfs +UNKEY_BUILDERD_WORKSPACE_DIR=/opt/builderd/workspace +UNKEY_BUILDERD_DATABASE_DATA_DIR=/opt/builderd/data + +# Disable observability for development +UNKEY_ASSETMANAGERD_OTEL_ENABLED=false +UNKEY_BILLAGED_OTEL_ENABLED=false +UNKEY_BUILDERD_OTEL_ENABLED=false +UNKEY_METALD_OTEL_ENABLED=false +EOF + +# Enable services (following LOCAL_DEPLOYMENT_GUIDE.md order) +RUN systemctl enable assetmanagerd.service && \ + systemctl enable billaged.service && \ + systemctl enable builderd.service && \ + systemctl enable metald.service + +# Create entrypoint script +RUN cat > /entrypoint.sh << 'EOF' +#!/bin/bash +set -e + +# Source environment variables +source /etc/default/unkey-deploy + +# Check Docker socket access +if [ -S /var/run/docker.sock ]; then + echo "Docker socket found, testing access..." + if timeout 10 docker version > /dev/null 2>&1; then + echo "Docker access confirmed" + else + echo "Warning: Docker socket exists but not accessible" + fi +else + echo "Warning: Docker socket not found at /var/run/docker.sock" +fi + +# Start services directly (systemd is problematic in containers) +echo "Starting services directly..." + +# Start services directly in background with proper ordering +echo "Starting assetmanagerd..." +/usr/local/bin/assetmanagerd & +ASSETMANAGERD_PID=$! + +echo "Starting billaged..." +/usr/local/bin/billaged & +BILLAGED_PID=$! + +echo "Starting builderd..." +/usr/local/bin/builderd & +BUILDERD_PID=$! + +# Wait a moment for dependencies to start +sleep 5 + +echo "Starting metald..." +/usr/local/bin/metald & +METALD_PID=$! + +echo "All services started. PIDs: assetmanagerd=$ASSETMANAGERD_PID billaged=$BILLAGED_PID builderd=$BUILDERD_PID metald=$METALD_PID" + +# Function to handle shutdown +shutdown_services() { + echo "Shutting down services..." + kill $METALD_PID $BUILDERD_PID $BILLAGED_PID $ASSETMANAGERD_PID 2>/dev/null || true + wait +} + +# Set up signal handler +trap shutdown_services SIGTERM SIGINT + +# Wait for all background processes +wait +EOF + +RUN chmod +x /entrypoint.sh + +# Expose service ports +EXPOSE 8080 8081 8082 8083 + +# Set up systemd as PID 1 +ENTRYPOINT ["/entrypoint.sh"] + +# Health check +HEALTHCHECK --interval=30s --timeout=10s --start-period=60s --retries=3 \ + CMD curl -f http://localhost:8080/health || exit 1 + +# Labels for identification +LABEL org.unkey.component="deploy-services" \ + org.unkey.version="dev" \ + org.unkey.description="Development environment for all Unkey deploy services" + +# AIDEV-NOTE: This Dockerfile follows the LOCAL_DEPLOYMENT_GUIDE.md as closely as possible +# Key features: +# 1. Uses Fedora 42 (production parity) +# 2. Multi-stage build with development tools +# 3. systemd as process manager +# 4. All services built using existing Makefiles +# 5. TLS disabled for development +# 6. Docker backend configured for metald +# 7. Proper directory structure and permissions diff --git a/go/deploy/README.dev.md b/go/deploy/README.dev.md new file mode 100644 index 0000000000..d1485cd487 --- /dev/null +++ b/go/deploy/README.dev.md @@ -0,0 +1,256 @@ +# Unkey Deploy Services - Development Environment + +This directory contains a containerized development environment for all Unkey deploy services. It provides a single Docker container running all four services (`assetmanagerd`, `billaged`, `builderd`, `metald`) with systemd as the process manager. + +## Quick Start + +```bash +# Navigate to deployment directory +cd deployment + +# Build and start all services (including deploy services) +docker-compose up -d + +# Check status +docker-compose ps + +# View deploy services logs +docker-compose logs -f metald-aio + +# Open shell in deploy container +docker exec -it metald-aio /bin/bash + +# Test services (from host) +metald-cli -tls-mode=disabled -server=http://localhost:8090 list + +# Test services (from inside container) +docker exec -it metald-aio metald-cli -tls-mode=disabled -server=http://localhost:8080 list +``` + +## Architecture + +### Single Container Design +- **Base Image**: Fedora 42 (production parity) +- **Process Manager**: systemd (production parity) +- **Services**: All 4 deploy services in one container +- **Backend**: Docker backend for metald (containers instead of VMs) + +### Service Ports +- `metald`: 8090 (exposed), 8080 (internal) +- `billaged`: 8081 (internal only) +- `builderd`: 8082 (internal only) +- `assetmanagerd`: 8083 (internal only) + +### Key Features +- **TLS Disabled**: All services use `TLS_MODE=disabled` for development +- **Docker Backend**: metald creates Docker containers instead of VMs +- **Production Parity**: Uses same systemd services and build process +- **Simplified Setup**: No Firecracker/KVM/SPIRE required + +## Files and Structure + +``` +go/deploy/ +├── Dockerfile.dev # Development container +├── docker-compose.dev.yml # Docker Compose configuration +├── Makefile.dev # Development commands +├── README.dev.md # This file +└── docker/ + └── systemd/ # systemd service files + ├── assetmanagerd.service + ├── billaged.service + ├── builderd.service + └── metald.service +``` + +## Usage + +### Starting Services + +```bash +# Build container (first time or after changes) +make -f Makefile.dev build + +# Start all services +make -f Makefile.dev up + +# Check if services are running +make -f Makefile.dev status +``` + +### Working with Services + +```bash +# View service logs +make -f Makefile.dev logs + +# Open shell in container +make -f Makefile.dev shell + +# Inside container, check systemd status +systemctl status assetmanagerd.service +systemctl status billaged.service +systemctl status builderd.service +systemctl status metald.service + +# View journal logs +journalctl -f -u metald.service +``` + +### Testing Services + +```bash +# Run basic health checks +make -f Makefile.dev test + +# Create a test VM (Docker container) +make -f Makefile.dev test-vm + +# Inside container, use CLI tools +assetmanagerd-cli -tls-mode=disabled -server=http://localhost:8083 list +billaged-cli -tls-mode=disabled -server=http://localhost:8081 health +builderd-cli -tls-mode=disabled -server=http://localhost:8082 health +metald-cli -tls-mode=disabled -server=http://localhost:8080 list +``` + +### Creating VMs (Docker Containers) + +```bash +# Inside container or using docker exec +metald-cli -tls-mode=disabled -server=http://localhost:8080 \ + -docker-image=alpine:latest create-and-boot + +# List VMs +metald-cli -tls-mode=disabled -server=http://localhost:8080 list + +# Get VM info +metald-cli -tls-mode=disabled -server=http://localhost:8080 info +``` + +## Configuration + +### Environment Variables + +The container uses `/etc/default/unkey-deploy` for configuration: + +```bash +# TLS disabled for development +UNKEY_ASSETMANAGERD_TLS_MODE=disabled +UNKEY_BILLAGED_TLS_MODE=disabled +UNKEY_BUILDERD_TLS_MODE=disabled +UNKEY_METALD_TLS_MODE=disabled + +# Docker backend for metald +UNKEY_METALD_BACKEND=docker + +# Service endpoints (HTTP) +UNKEY_METALD_BILLING_ENDPOINT=http://localhost:8081 +UNKEY_METALD_ASSETMANAGER_ENDPOINT=http://localhost:8083 +UNKEY_ASSETMANAGERD_BUILDERD_ENDPOINT=http://localhost:8082 +UNKEY_BUILDERD_ASSETMANAGER_ENDPOINT=http://localhost:8083 +``` + +### Docker Backend Configuration + +metald uses Docker backend with these settings: +- **Docker Host**: `unix:///var/run/docker.sock` +- **Network**: `bridge` +- **Port Range**: 30000-40000 +- **Container Prefix**: `unkey-vm-` + +## Troubleshooting + +### Services Not Starting + +```bash +# Check systemd status +make -f Makefile.dev shell +systemctl status assetmanagerd.service +systemctl status billaged.service +systemctl status builderd.service +systemctl status metald.service + +# Check logs +journalctl -u metald.service +journalctl -u assetmanagerd.service +``` + +### Docker Access Issues + +```bash +# Check Docker socket access +docker exec unkey-deploy-dev ls -la /var/run/docker.sock +docker exec unkey-deploy-dev docker version + +# Ensure Docker socket is mounted +docker-compose -f docker-compose.dev.yml config +``` + +### Port Conflicts + +```bash +# Check if ports are available +netstat -tlnp | grep -E ':(8080|8081|8082|8083)' + +# Use different ports if needed +docker-compose -f docker-compose.dev.yml down +# Edit docker-compose.dev.yml ports section +docker-compose -f docker-compose.dev.yml up -d +``` + +## Development Workflow + +### Making Changes + +1. Edit service code +2. Rebuild container: `make -f Makefile.dev build` +3. Restart services: `make -f Makefile.dev down up` +4. Test changes: `make -f Makefile.dev test` + +### Debugging + +```bash +# Enter container +make -f Makefile.dev shell + +# Check service logs +journalctl -f -u metald.service + +# Test individual services +assetmanagerd-cli -tls-mode=disabled -server=http://localhost:8083 list +metald-cli -tls-mode=disabled -server=http://localhost:8080 list + +# Check Docker containers created by metald +docker ps --filter "label=unkey.vm.created_by=metald" +``` + +## Integration with Main Docker Compose + +This development environment is designed to work alongside the main `deployment/docker-compose.yaml`. The services will be available at: + +- **Web services**: Ports 3000, 8787, etc. (from main compose) +- **Deploy services**: Ports 8080-8083 (from this dev environment) + +Both environments share the same Docker network for seamless integration. + +## Benefits + +1. **Production Parity**: Uses Fedora + systemd + same build process +2. **Simplified Development**: No complex VM setup required +3. **Fast Iteration**: Instant container startup vs VM boot times +4. **Familiar Tools**: Standard Docker debugging and monitoring +5. **Easy Integration**: Works with existing docker-compose setup + +## Limitations + +1. **Single Container**: All services in one container (less isolation) +2. **Development Only**: TLS disabled, simplified configuration +3. **Docker Dependency**: Requires Docker socket access +4. **systemd Overhead**: Requires privileged container for systemd + +## Next Steps + +- [ ] Add hot reload for development +- [ ] Integrate with VS Code dev containers +- [ ] Add performance monitoring +- [ ] Create CI/CD pipeline for testing \ No newline at end of file diff --git a/go/deploy/assetmanagerd/client/client.go b/go/deploy/assetmanagerd/client/client.go index 20d4861b57..a64d9d20a9 100644 --- a/go/deploy/assetmanagerd/client/client.go +++ b/go/deploy/assetmanagerd/client/client.go @@ -7,8 +7,8 @@ import ( "time" "connectrpc.com/connect" - assetv1 "github.com/unkeyed/unkey/go/deploy/assetmanagerd/gen/asset/v1" - "github.com/unkeyed/unkey/go/deploy/assetmanagerd/gen/asset/v1/assetv1connect" + assetv1 "github.com/unkeyed/unkey/go/gen/proto/deploy/assetmanagerd/v1" + "github.com/unkeyed/unkey/go/gen/proto/deploy/assetmanagerd/v1/assetmanagerdv1connect" "github.com/unkeyed/unkey/go/deploy/pkg/tls" ) @@ -41,7 +41,7 @@ type Config struct { // Client provides a high-level interface to assetmanagerd services type Client struct { - assetService assetv1connect.AssetManagerServiceClient + assetService assetmanagerdv1connect.AssetManagerServiceClient tlsProvider tls.Provider userID string tenantID string @@ -92,7 +92,7 @@ func New(ctx context.Context, config Config) (*Client, error) { } // Create ConnectRPC client - assetService := assetv1connect.NewAssetManagerServiceClient( + assetService := assetmanagerdv1connect.NewAssetManagerServiceClient( httpClient, config.ServerAddress, ) diff --git a/go/deploy/assetmanagerd/client/types.go b/go/deploy/assetmanagerd/client/types.go index 5bddea9130..9e74fb62ac 100644 --- a/go/deploy/assetmanagerd/client/types.go +++ b/go/deploy/assetmanagerd/client/types.go @@ -1,7 +1,7 @@ package client import ( - assetv1 "github.com/unkeyed/unkey/go/deploy/assetmanagerd/gen/asset/v1" + assetv1 "github.com/unkeyed/unkey/go/gen/proto/deploy/assetmanagerd/v1" ) // AIDEV-NOTE: Type definitions for assetmanagerd client requests and responses diff --git a/go/deploy/assetmanagerd/cmd/assetmanagerd-cli/main.go b/go/deploy/assetmanagerd/cmd/assetmanagerd-cli/main.go index 04f4990325..bb7d2f9ff1 100644 --- a/go/deploy/assetmanagerd/cmd/assetmanagerd-cli/main.go +++ b/go/deploy/assetmanagerd/cmd/assetmanagerd-cli/main.go @@ -10,7 +10,7 @@ import ( "time" "github.com/unkeyed/unkey/go/deploy/assetmanagerd/client" - assetv1 "github.com/unkeyed/unkey/go/deploy/assetmanagerd/gen/asset/v1" + assetv1 "github.com/unkeyed/unkey/go/gen/proto/deploy/assetmanagerd/v1" ) // AIDEV-NOTE: CLI tool demonstrating assetmanagerd client usage with SPIFFE integration diff --git a/go/deploy/assetmanagerd/cmd/assetmanagerd/main.go b/go/deploy/assetmanagerd/cmd/assetmanagerd/main.go index 5785f9d04b..946be34de9 100644 --- a/go/deploy/assetmanagerd/cmd/assetmanagerd/main.go +++ b/go/deploy/assetmanagerd/cmd/assetmanagerd/main.go @@ -16,8 +16,8 @@ import ( "time" "connectrpc.com/connect" - assetv1 "github.com/unkeyed/unkey/go/deploy/assetmanagerd/gen/asset/v1" - "github.com/unkeyed/unkey/go/deploy/assetmanagerd/gen/asset/v1/assetv1connect" + assetv1 "github.com/unkeyed/unkey/go/gen/proto/deploy/assetmanagerd/v1" + "github.com/unkeyed/unkey/go/gen/proto/deploy/assetmanagerd/v1/assetmanagerdv1connect" "github.com/unkeyed/unkey/go/deploy/assetmanagerd/internal/builderd" "github.com/unkeyed/unkey/go/deploy/assetmanagerd/internal/config" "github.com/unkeyed/unkey/go/deploy/assetmanagerd/internal/observability" @@ -227,7 +227,7 @@ func main() { } // Create ConnectRPC handler with shared interceptors - path, handler := assetv1connect.NewAssetManagerServiceHandler( + path, handler := assetmanagerdv1connect.NewAssetManagerServiceHandler( assetService, connect.WithInterceptors(interceptorList...), ) diff --git a/go/deploy/assetmanagerd/internal/builderd/client.go b/go/deploy/assetmanagerd/internal/builderd/client.go index a68cb8b432..abe82ff5e8 100644 --- a/go/deploy/assetmanagerd/internal/builderd/client.go +++ b/go/deploy/assetmanagerd/internal/builderd/client.go @@ -7,8 +7,8 @@ import ( "time" "connectrpc.com/connect" - builderv1 "github.com/unkeyed/unkey/go/deploy/builderd/gen/builder/v1" - "github.com/unkeyed/unkey/go/deploy/builderd/gen/builder/v1/builderv1connect" + builderv1 "github.com/unkeyed/unkey/go/gen/proto/deploy/builderd/v1" + "github.com/unkeyed/unkey/go/gen/proto/deploy/builderd/v1/builderdv1connect" tlspkg "github.com/unkeyed/unkey/go/deploy/pkg/tls" "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" ) @@ -63,7 +63,7 @@ type CompletedBuild struct { type Client struct { cfg *Config logger *slog.Logger - builderClient builderv1connect.BuilderServiceClient + builderClient builderdv1connect.BuilderServiceClient } // NewClient creates a new builderd client @@ -75,7 +75,7 @@ func NewClient(cfg *Config, logger *slog.Logger) (*Client, error) { httpClient.Transport = otelhttp.NewTransport(httpClient.Transport) // Create Connect client - builderClient := builderv1connect.NewBuilderServiceClient( + builderClient := builderdv1connect.NewBuilderServiceClient( httpClient, cfg.Endpoint, ) diff --git a/go/deploy/assetmanagerd/internal/registry/registry.go b/go/deploy/assetmanagerd/internal/registry/registry.go index 04685bc392..904b0a0ebc 100644 --- a/go/deploy/assetmanagerd/internal/registry/registry.go +++ b/go/deploy/assetmanagerd/internal/registry/registry.go @@ -10,7 +10,7 @@ import ( _ "github.com/mattn/go-sqlite3" "github.com/oklog/ulid/v2" - assetv1 "github.com/unkeyed/unkey/go/deploy/assetmanagerd/gen/asset/v1" + assetv1 "github.com/unkeyed/unkey/go/gen/proto/deploy/assetmanagerd/v1" ) // Registry manages asset metadata in SQLite diff --git a/go/deploy/assetmanagerd/internal/service/service.go b/go/deploy/assetmanagerd/internal/service/service.go index e59a583840..b8dd3911a6 100644 --- a/go/deploy/assetmanagerd/internal/service/service.go +++ b/go/deploy/assetmanagerd/internal/service/service.go @@ -11,7 +11,7 @@ import ( "time" "connectrpc.com/connect" - assetv1 "github.com/unkeyed/unkey/go/deploy/assetmanagerd/gen/asset/v1" + assetv1 "github.com/unkeyed/unkey/go/gen/proto/deploy/assetmanagerd/v1" "github.com/unkeyed/unkey/go/deploy/assetmanagerd/internal/builderd" "github.com/unkeyed/unkey/go/deploy/assetmanagerd/internal/config" "github.com/unkeyed/unkey/go/deploy/assetmanagerd/internal/registry" diff --git a/go/deploy/billaged/client/client.go b/go/deploy/billaged/client/client.go index 25251373f3..369f020264 100644 --- a/go/deploy/billaged/client/client.go +++ b/go/deploy/billaged/client/client.go @@ -7,8 +7,8 @@ import ( "time" "connectrpc.com/connect" - billingv1 "github.com/unkeyed/unkey/go/deploy/billaged/gen/billing/v1" - "github.com/unkeyed/unkey/go/deploy/billaged/gen/billing/v1/billingv1connect" + billingv1 "github.com/unkeyed/unkey/go/gen/proto/deploy/billaged/v1" + "github.com/unkeyed/unkey/go/gen/proto/deploy/billaged/v1/billagedv1connect" "github.com/unkeyed/unkey/go/deploy/pkg/tls" ) @@ -41,7 +41,7 @@ type Config struct { // Client provides a high-level interface to billaged services type Client struct { - billingService billingv1connect.BillingServiceClient + billingService billagedv1connect.BillingServiceClient tlsProvider tls.Provider userID string tenantID string @@ -92,7 +92,7 @@ func New(ctx context.Context, config Config) (*Client, error) { } // Create ConnectRPC client - billingService := billingv1connect.NewBillingServiceClient( + billingService := billagedv1connect.NewBillingServiceClient( httpClient, config.ServerAddress, ) diff --git a/go/deploy/billaged/client/types.go b/go/deploy/billaged/client/types.go index acae307cd8..ad26037d8b 100644 --- a/go/deploy/billaged/client/types.go +++ b/go/deploy/billaged/client/types.go @@ -1,7 +1,7 @@ package client import ( - billingv1 "github.com/unkeyed/unkey/go/deploy/billaged/gen/billing/v1" + billingv1 "github.com/unkeyed/unkey/go/gen/proto/deploy/billaged/v1" ) // AIDEV-NOTE: Type definitions for billaged client requests and responses diff --git a/go/deploy/billaged/cmd/billaged-cli/main.go b/go/deploy/billaged/cmd/billaged-cli/main.go index 04ee0bcc27..39fc963604 100644 --- a/go/deploy/billaged/cmd/billaged-cli/main.go +++ b/go/deploy/billaged/cmd/billaged-cli/main.go @@ -11,7 +11,7 @@ import ( "time" "github.com/unkeyed/unkey/go/deploy/billaged/client" - billingv1 "github.com/unkeyed/unkey/go/deploy/billaged/gen/billing/v1" + billingv1 "github.com/unkeyed/unkey/go/gen/proto/deploy/billaged/v1" "google.golang.org/protobuf/types/known/timestamppb" ) diff --git a/go/deploy/billaged/cmd/billaged/main.go b/go/deploy/billaged/cmd/billaged/main.go index 7a25f15624..dcbcc511f9 100644 --- a/go/deploy/billaged/cmd/billaged/main.go +++ b/go/deploy/billaged/cmd/billaged/main.go @@ -15,7 +15,7 @@ import ( "connectrpc.com/connect" "github.com/prometheus/client_golang/prometheus/promhttp" - "github.com/unkeyed/unkey/go/deploy/billaged/gen/billing/v1/billingv1connect" + "github.com/unkeyed/unkey/go/gen/proto/deploy/billaged/v1/billagedv1connect" "github.com/unkeyed/unkey/go/deploy/billaged/internal/aggregator" "github.com/unkeyed/unkey/go/deploy/billaged/internal/config" "github.com/unkeyed/unkey/go/deploy/billaged/internal/observability" @@ -230,7 +230,7 @@ func main() { } mux := http.NewServeMux() - path, handler := billingv1connect.NewBillingServiceHandler(billingService, + path, handler := billagedv1connect.NewBillingServiceHandler(billingService, connect.WithInterceptors(interceptorList...), ) mux.Handle(path, handler) diff --git a/go/deploy/billaged/internal/aggregator/aggregator.go b/go/deploy/billaged/internal/aggregator/aggregator.go index 8a018c3966..cc1379617d 100644 --- a/go/deploy/billaged/internal/aggregator/aggregator.go +++ b/go/deploy/billaged/internal/aggregator/aggregator.go @@ -6,7 +6,7 @@ import ( "sync" "time" - billingv1 "github.com/unkeyed/unkey/go/deploy/billaged/gen/billing/v1" + billingv1 "github.com/unkeyed/unkey/go/gen/proto/deploy/billaged/v1" ) // VMUsageData tracks usage for a single VM diff --git a/go/deploy/billaged/internal/service/billing.go b/go/deploy/billaged/internal/service/billing.go index 85af6914d8..683494b880 100644 --- a/go/deploy/billaged/internal/service/billing.go +++ b/go/deploy/billaged/internal/service/billing.go @@ -7,8 +7,8 @@ import ( "time" "connectrpc.com/connect" - billingv1 "github.com/unkeyed/unkey/go/deploy/billaged/gen/billing/v1" - "github.com/unkeyed/unkey/go/deploy/billaged/gen/billing/v1/billingv1connect" + billingv1 "github.com/unkeyed/unkey/go/gen/proto/deploy/billaged/v1" + "github.com/unkeyed/unkey/go/gen/proto/deploy/billaged/v1/billagedv1connect" "github.com/unkeyed/unkey/go/deploy/billaged/internal/aggregator" "github.com/unkeyed/unkey/go/deploy/billaged/internal/observability" ) @@ -176,4 +176,4 @@ func (s *BillingService) NotifyPossibleGap( } // Ensure BillingService implements the interface -var _ billingv1connect.BillingServiceHandler = (*BillingService)(nil) +var _ billagedv1connect.BillingServiceHandler = (*BillingService)(nil) diff --git a/go/deploy/builderd/client/client.go b/go/deploy/builderd/client/client.go index 6789f41e48..faadfbed1f 100644 --- a/go/deploy/builderd/client/client.go +++ b/go/deploy/builderd/client/client.go @@ -7,8 +7,8 @@ import ( "time" "connectrpc.com/connect" - builderv1 "github.com/unkeyed/unkey/go/deploy/builderd/gen/builder/v1" - "github.com/unkeyed/unkey/go/deploy/builderd/gen/builder/v1/builderv1connect" + builderv1 "github.com/unkeyed/unkey/go/gen/proto/deploy/builderd/v1" + "github.com/unkeyed/unkey/go/gen/proto/deploy/builderd/v1/builderdv1connect" "github.com/unkeyed/unkey/go/deploy/pkg/tls" ) @@ -41,7 +41,7 @@ type Config struct { // Client provides a high-level interface to builderd services type Client struct { - builderService builderv1connect.BuilderServiceClient + builderService builderdv1connect.BuilderServiceClient tlsProvider tls.Provider userID string tenantID string @@ -92,7 +92,7 @@ func New(ctx context.Context, config Config) (*Client, error) { } // Create ConnectRPC client - builderService := builderv1connect.NewBuilderServiceClient( + builderService := builderdv1connect.NewBuilderServiceClient( httpClient, config.ServerAddress, ) diff --git a/go/deploy/builderd/client/types.go b/go/deploy/builderd/client/types.go index 049802b3e2..0fe8f4da33 100644 --- a/go/deploy/builderd/client/types.go +++ b/go/deploy/builderd/client/types.go @@ -1,7 +1,7 @@ package client import ( - builderv1 "github.com/unkeyed/unkey/go/deploy/builderd/gen/builder/v1" + builderv1 "github.com/unkeyed/unkey/go/gen/proto/deploy/builderd/v1" "google.golang.org/protobuf/types/known/timestamppb" ) diff --git a/go/deploy/builderd/cmd/builderd-cli/main.go b/go/deploy/builderd/cmd/builderd-cli/main.go index 3a65a828b5..016ed7767c 100644 --- a/go/deploy/builderd/cmd/builderd-cli/main.go +++ b/go/deploy/builderd/cmd/builderd-cli/main.go @@ -11,7 +11,7 @@ import ( "time" "github.com/unkeyed/unkey/go/deploy/builderd/client" - builderv1 "github.com/unkeyed/unkey/go/deploy/builderd/gen/builder/v1" + builderv1 "github.com/unkeyed/unkey/go/gen/proto/deploy/builderd/v1" "google.golang.org/protobuf/types/known/timestamppb" ) diff --git a/go/deploy/builderd/cmd/builderd/main.go b/go/deploy/builderd/cmd/builderd/main.go index 068e44b432..c26ec479ae 100644 --- a/go/deploy/builderd/cmd/builderd/main.go +++ b/go/deploy/builderd/cmd/builderd/main.go @@ -19,8 +19,8 @@ import ( "connectrpc.com/connect" "github.com/prometheus/client_golang/prometheus/promhttp" - assetv1 "github.com/unkeyed/unkey/go/deploy/assetmanagerd/gen/asset/v1" - "github.com/unkeyed/unkey/go/deploy/builderd/gen/builder/v1/builderv1connect" + assetv1 "github.com/unkeyed/unkey/go/gen/proto/deploy/assetmanagerd/v1" + "github.com/unkeyed/unkey/go/gen/proto/deploy/builderd/v1/builderdv1connect" "github.com/unkeyed/unkey/go/deploy/builderd/internal/assetmanager" "github.com/unkeyed/unkey/go/deploy/builderd/internal/config" "github.com/unkeyed/unkey/go/deploy/builderd/internal/observability" @@ -264,7 +264,7 @@ func main() { } mux := http.NewServeMux() - path, handler := builderv1connect.NewBuilderServiceHandler(builderService, + path, handler := builderdv1connect.NewBuilderServiceHandler(builderService, connect.WithInterceptors(interceptorList...), ) mux.Handle(path, handler) diff --git a/go/deploy/builderd/internal/assetmanager/client.go b/go/deploy/builderd/internal/assetmanager/client.go index 4f9ddc0413..ecf122467b 100644 --- a/go/deploy/builderd/internal/assetmanager/client.go +++ b/go/deploy/builderd/internal/assetmanager/client.go @@ -9,8 +9,8 @@ import ( "os" "path/filepath" - assetv1 "github.com/unkeyed/unkey/go/deploy/assetmanagerd/gen/asset/v1" - "github.com/unkeyed/unkey/go/deploy/assetmanagerd/gen/asset/v1/assetv1connect" + assetv1 "github.com/unkeyed/unkey/go/gen/proto/deploy/assetmanagerd/v1" + "github.com/unkeyed/unkey/go/gen/proto/deploy/assetmanagerd/v1/assetmanagerdv1connect" "github.com/unkeyed/unkey/go/deploy/builderd/internal/config" "github.com/unkeyed/unkey/go/deploy/pkg/tls" "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" @@ -18,7 +18,7 @@ import ( // Client provides access to the assetmanagerd service type Client struct { - client assetv1connect.AssetManagerServiceClient + client assetmanagerdv1connect.AssetManagerServiceClient logger *slog.Logger enabled bool endpoint string @@ -43,7 +43,7 @@ func NewClient(cfg *config.Config, logger *slog.Logger, tlsProvider tls.Provider httpClient.Transport = otelhttp.NewTransport(httpClient.Transport) // Create Connect client - client := assetv1connect.NewAssetManagerServiceClient( + client := assetmanagerdv1connect.NewAssetManagerServiceClient( httpClient, cfg.AssetManager.Endpoint, ) diff --git a/go/deploy/builderd/internal/assets/base.go b/go/deploy/builderd/internal/assets/base.go index 03baaef064..e7ebf29ad1 100644 --- a/go/deploy/builderd/internal/assets/base.go +++ b/go/deploy/builderd/internal/assets/base.go @@ -10,7 +10,7 @@ import ( "os" "path/filepath" - assetv1 "github.com/unkeyed/unkey/go/deploy/assetmanagerd/gen/asset/v1" + assetv1 "github.com/unkeyed/unkey/go/gen/proto/deploy/assetmanagerd/v1" "github.com/unkeyed/unkey/go/deploy/builderd/internal/assetmanager" "github.com/unkeyed/unkey/go/deploy/builderd/internal/config" ) diff --git a/go/deploy/builderd/internal/executor/docker.go b/go/deploy/builderd/internal/executor/docker.go index 2707847127..7a55f71a3c 100644 --- a/go/deploy/builderd/internal/executor/docker.go +++ b/go/deploy/builderd/internal/executor/docker.go @@ -16,7 +16,7 @@ import ( "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" - builderv1 "github.com/unkeyed/unkey/go/deploy/builderd/gen/builder/v1" + builderv1 "github.com/unkeyed/unkey/go/gen/proto/deploy/builderd/v1" "github.com/unkeyed/unkey/go/deploy/builderd/internal/config" "github.com/unkeyed/unkey/go/deploy/builderd/internal/observability" "github.com/unkeyed/unkey/go/deploy/pkg/observability/interceptors" diff --git a/go/deploy/builderd/internal/executor/registry.go b/go/deploy/builderd/internal/executor/registry.go index 69c51dbe3f..be308ab10f 100644 --- a/go/deploy/builderd/internal/executor/registry.go +++ b/go/deploy/builderd/internal/executor/registry.go @@ -6,7 +6,7 @@ import ( "log/slog" "sync" - builderv1 "github.com/unkeyed/unkey/go/deploy/builderd/gen/builder/v1" + builderv1 "github.com/unkeyed/unkey/go/gen/proto/deploy/builderd/v1" "github.com/unkeyed/unkey/go/deploy/builderd/internal/config" "github.com/unkeyed/unkey/go/deploy/builderd/internal/observability" ) diff --git a/go/deploy/builderd/internal/executor/types.go b/go/deploy/builderd/internal/executor/types.go index 09e5c0b1c4..ba02bc813a 100644 --- a/go/deploy/builderd/internal/executor/types.go +++ b/go/deploy/builderd/internal/executor/types.go @@ -4,7 +4,7 @@ import ( "context" "time" - builderv1 "github.com/unkeyed/unkey/go/deploy/builderd/gen/builder/v1" + builderv1 "github.com/unkeyed/unkey/go/gen/proto/deploy/builderd/v1" ) // Executor defines the interface for build executors diff --git a/go/deploy/builderd/internal/service/builder.go b/go/deploy/builderd/internal/service/builder.go index 2373df8dca..cf1826f4a6 100644 --- a/go/deploy/builderd/internal/service/builder.go +++ b/go/deploy/builderd/internal/service/builder.go @@ -11,8 +11,8 @@ import ( "time" "connectrpc.com/connect" - assetv1 "github.com/unkeyed/unkey/go/deploy/assetmanagerd/gen/asset/v1" - builderv1 "github.com/unkeyed/unkey/go/deploy/builderd/gen/builder/v1" + assetv1 "github.com/unkeyed/unkey/go/gen/proto/deploy/assetmanagerd/v1" + builderv1 "github.com/unkeyed/unkey/go/gen/proto/deploy/builderd/v1" "github.com/unkeyed/unkey/go/deploy/builderd/internal/assetmanager" "github.com/unkeyed/unkey/go/deploy/builderd/internal/config" "github.com/unkeyed/unkey/go/deploy/builderd/internal/executor" diff --git a/go/deploy/builderd/internal/tenant/isolation.go b/go/deploy/builderd/internal/tenant/isolation.go index 4d1918d27a..4a349939fc 100644 --- a/go/deploy/builderd/internal/tenant/isolation.go +++ b/go/deploy/builderd/internal/tenant/isolation.go @@ -12,7 +12,7 @@ import ( "syscall" "time" - builderv1 "github.com/unkeyed/unkey/go/deploy/builderd/gen/builder/v1" + builderv1 "github.com/unkeyed/unkey/go/gen/proto/deploy/builderd/v1" ) const ( diff --git a/go/deploy/builderd/internal/tenant/manager.go b/go/deploy/builderd/internal/tenant/manager.go index f63f636042..485807faf2 100644 --- a/go/deploy/builderd/internal/tenant/manager.go +++ b/go/deploy/builderd/internal/tenant/manager.go @@ -7,7 +7,7 @@ import ( "sync" "time" - builderv1 "github.com/unkeyed/unkey/go/deploy/builderd/gen/builder/v1" + builderv1 "github.com/unkeyed/unkey/go/gen/proto/deploy/builderd/v1" "github.com/unkeyed/unkey/go/deploy/builderd/internal/config" ) diff --git a/go/deploy/builderd/internal/tenant/storage.go b/go/deploy/builderd/internal/tenant/storage.go index cb86cbc8ab..d228686f85 100644 --- a/go/deploy/builderd/internal/tenant/storage.go +++ b/go/deploy/builderd/internal/tenant/storage.go @@ -15,7 +15,7 @@ import ( "strings" "time" - builderv1 "github.com/unkeyed/unkey/go/deploy/builderd/gen/builder/v1" + builderv1 "github.com/unkeyed/unkey/go/gen/proto/deploy/builderd/v1" ) // StorageIsolator handles storage isolation and encryption for tenants diff --git a/go/deploy/docker/systemd/assetmanagerd.service b/go/deploy/docker/systemd/assetmanagerd.service new file mode 100644 index 0000000000..ffd69d324e --- /dev/null +++ b/go/deploy/docker/systemd/assetmanagerd.service @@ -0,0 +1,56 @@ +[Unit] +Description=AssetManagerd VM Asset Management Service (Development) +Documentation=https://github.com/unkeyed/unkey/go/deploy/assetmanagerd +After=network.target +Wants=network.target + +[Service] +Type=simple +# Run as root for development simplicity +User=root +Group=root +# Load development environment variables +EnvironmentFile=/etc/default/unkey-deploy +# Service will start in the required directories +ExecStart=/usr/local/bin/assetmanagerd +Restart=always +RestartSec=5 +StandardOutput=journal +StandardError=journal +SyslogIdentifier=assetmanagerd + +# Development-specific environment overrides +Environment=UNKEY_ASSETMANAGERD_TLS_MODE=disabled +Environment=UNKEY_ASSETMANAGERD_OTEL_ENABLED=false +Environment=UNKEY_ASSETMANAGERD_PORT=8083 +Environment=UNKEY_ASSETMANAGERD_ADDRESS=0.0.0.0 + +# Storage configuration for development +Environment=UNKEY_ASSETMANAGERD_STORAGE_BACKEND=local +Environment=UNKEY_ASSETMANAGERD_LOCAL_STORAGE_PATH=/opt/builderd/rootfs +Environment=UNKEY_ASSETMANAGERD_DATABASE_PATH=/opt/assetmanagerd/assets.db +Environment=UNKEY_ASSETMANAGERD_CACHE_DIR=/opt/assetmanagerd/cache + +# Garbage collection +Environment=UNKEY_ASSETMANAGERD_GC_ENABLED=true +Environment=UNKEY_ASSETMANAGERD_GC_INTERVAL=1h +Environment=UNKEY_ASSETMANAGERD_GC_MAX_AGE=24h + +# Builderd integration configuration +Environment=UNKEY_ASSETMANAGERD_BUILDERD_ENABLED=true +Environment=UNKEY_ASSETMANAGERD_BUILDERD_ENDPOINT=http://localhost:8082 +Environment=UNKEY_ASSETMANAGERD_BUILDERD_TIMEOUT=30m +Environment=UNKEY_ASSETMANAGERD_BUILDERD_AUTO_REGISTER=true +Environment=UNKEY_ASSETMANAGERD_BUILDERD_MAX_RETRIES=3 +Environment=UNKEY_ASSETMANAGERD_BUILDERD_RETRY_DELAY=5s + +# Resource limits +LimitNOFILE=65536 +LimitNPROC=4096 + +# Basic security settings +NoNewPrivileges=true +PrivateTmp=true + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/go/deploy/docker/systemd/billaged.service b/go/deploy/docker/systemd/billaged.service new file mode 100644 index 0000000000..863246f251 --- /dev/null +++ b/go/deploy/docker/systemd/billaged.service @@ -0,0 +1,33 @@ +[Unit] +Description=Billaged VM Usage Billing Service (Development) +Documentation=https://github.com/unkeyed/unkey/go/deploy/billaged +After=network.target +Wants=network.target + +[Service] +Type=simple +User=billaged +Group=billaged +# Load development environment variables +EnvironmentFile=/etc/default/unkey-deploy +# Service will start in the required directories +ExecStart=/usr/local/bin/billaged +Restart=always +RestartSec=5 +StandardOutput=journal +StandardError=journal +SyslogIdentifier=billaged + +# Development-specific environment overrides +Environment=UNKEY_BILLAGED_TLS_MODE=disabled +Environment=UNKEY_BILLAGED_OTEL_ENABLED=false +Environment=UNKEY_BILLAGED_PORT=8081 +Environment=UNKEY_BILLAGED_ADDRESS=0.0.0.0 +Environment=UNKEY_BILLAGED_AGGREGATION_INTERVAL=60s + +# Resource limits +LimitNOFILE=65536 +LimitNPROC=4096 + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/go/deploy/docker/systemd/builderd.service b/go/deploy/docker/systemd/builderd.service new file mode 100644 index 0000000000..67b0bd93d0 --- /dev/null +++ b/go/deploy/docker/systemd/builderd.service @@ -0,0 +1,62 @@ +[Unit] +Description=Builderd Multi-Tenant Build Service (Development) +Documentation=https://github.com/unkeyed/unkey/go/deploy/builderd +After=network.target +Wants=network.target + +[Service] +Type=simple +# Run as root for Docker access +User=root +Group=root +# Load development environment variables +EnvironmentFile=/etc/default/unkey-deploy +# Service will start in the required directories +ExecStart=/usr/local/bin/builderd +Restart=always +RestartSec=5 +StandardOutput=journal +StandardError=journal +SyslogIdentifier=builderd + +# Development-specific environment overrides +Environment=UNKEY_BUILDERD_TLS_MODE=disabled +Environment=UNKEY_BUILDERD_OTEL_ENABLED=false +Environment=UNKEY_BUILDERD_PORT=8082 +Environment=UNKEY_BUILDERD_ADDRESS=0.0.0.0 + +# Build configuration for development +Environment=UNKEY_BUILDERD_MAX_CONCURRENT_BUILDS=3 +Environment=UNKEY_BUILDERD_BUILD_TIMEOUT=15m +Environment=UNKEY_BUILDERD_SCRATCH_DIR=/opt/builderd/scratch +Environment=UNKEY_BUILDERD_ROOTFS_OUTPUT_DIR=/opt/builderd/rootfs +Environment=UNKEY_BUILDERD_WORKSPACE_DIR=/opt/builderd/workspace + +# Storage configuration +Environment=UNKEY_BUILDERD_STORAGE_BACKEND=local +Environment=UNKEY_BUILDERD_STORAGE_RETENTION_DAYS=7 + +# Database configuration +Environment=UNKEY_BUILDERD_DATABASE_TYPE=sqlite +Environment=UNKEY_BUILDERD_DATABASE_DATA_DIR=/opt/builderd/data + +# Docker configuration +Environment=UNKEY_BUILDERD_DOCKER_MAX_IMAGE_SIZE_GB=2 + +# Tenant isolation (disabled for development) +Environment=UNKEY_BUILDERD_TENANT_ISOLATION_ENABLED=false + +# AssetManagerd integration +Environment=UNKEY_BUILDERD_ASSETMANAGER_ENABLED=true +Environment=UNKEY_BUILDERD_ASSETMANAGER_ENDPOINT=http://localhost:8083 + +# Resource limits +LimitNOFILE=65536 +LimitNPROC=4096 + +# Shutdown configuration +TimeoutStopSec=30 +KillMode=mixed + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/go/deploy/docker/systemd/metald.service b/go/deploy/docker/systemd/metald.service new file mode 100644 index 0000000000..5d3627ccfe --- /dev/null +++ b/go/deploy/docker/systemd/metald.service @@ -0,0 +1,58 @@ +[Unit] +Description=Metald VM Management Service (Development) +Documentation=https://github.com/unkeyed/unkey/go/deploy/metald +After=network.target assetmanagerd.service billaged.service +Wants=network.target +# Start after assetmanagerd and billaged for proper service order + +[Service] +Type=simple +# Run as root for Docker access +User=root +Group=root +# Load development environment variables +EnvironmentFile=/etc/default/unkey-deploy +# Service will start in the required directories +ExecStart=/usr/local/bin/metald +Restart=always +RestartSec=5 +StandardOutput=journal +StandardError=journal +SyslogIdentifier=metald + +# Development-specific environment overrides +Environment=UNKEY_METALD_TLS_MODE=disabled +Environment=UNKEY_METALD_OTEL_ENABLED=false +Environment=UNKEY_METALD_PORT=8080 +Environment=UNKEY_METALD_ADDRESS=0.0.0.0 + +# Docker backend configuration +Environment=UNKEY_METALD_BACKEND=docker +Environment=UNKEY_METALD_DOCKER_HOST=unix:///var/run/docker.sock + +# Process Manager Configuration +Environment=UNKEY_METALD_SOCKET_DIR=/opt/metald/sockets +Environment=UNKEY_METALD_LOG_DIR=/opt/metald/logs +Environment=UNKEY_METALD_MAX_PROCESSES=100 + +# Billing Configuration +Environment=UNKEY_METALD_BILLING_ENABLED=true +Environment=UNKEY_METALD_BILLING_ENDPOINT=http://localhost:8081 +Environment=UNKEY_METALD_BILLING_MOCK_MODE=false + +# AssetManager Configuration +Environment=UNKEY_METALD_ASSETMANAGER_ENABLED=true +Environment=UNKEY_METALD_ASSETMANAGER_ENDPOINT=http://localhost:8083 +Environment=UNKEY_METALD_ASSETMANAGER_CACHE_DIR=/opt/metald/assets + +# Network configuration for Docker backend +Environment=UNKEY_METALD_DOCKER_NETWORK=bridge +Environment=UNKEY_METALD_DOCKER_PORT_RANGE_MIN=30000 +Environment=UNKEY_METALD_DOCKER_PORT_RANGE_MAX=40000 + +# Resource limits +LimitNOFILE=65536 +LimitNPROC=4096 + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/go/deploy/metald/cmd/metald/main.go b/go/deploy/metald/cmd/metald/main.go index fc4b66d102..7d51128d3f 100644 --- a/go/deploy/metald/cmd/metald/main.go +++ b/go/deploy/metald/cmd/metald/main.go @@ -15,6 +15,7 @@ import ( "time" "github.com/unkeyed/unkey/go/deploy/metald/internal/assetmanager" + "github.com/unkeyed/unkey/go/deploy/metald/internal/backend/docker" "github.com/unkeyed/unkey/go/deploy/metald/internal/backend/firecracker" "github.com/unkeyed/unkey/go/deploy/metald/internal/backend/types" "github.com/unkeyed/unkey/go/deploy/metald/internal/billing" @@ -280,6 +281,20 @@ func main() { backend = sdkClient // Note: Network manager is initialized and managed by SDK v4 + case types.BackendTypeDocker: + // AIDEV-NOTE: Docker backend for development - creates containers instead of VMs + logger.Info("initializing Docker backend for development") + + dockerClient, err := docker.NewDockerBackend(logger, docker.DefaultDockerBackendConfig()) + if err != nil { + logger.Error("failed to create Docker backend", + slog.String("error", err.Error()), + ) + os.Exit(1) + } + + backend = dockerClient + logger.Info("Docker backend initialized successfully") case types.BackendTypeCloudHypervisor: logger.Error("CloudHypervisor backend not implemented", slog.String("backend", string(cfg.Backend.Type)), diff --git a/go/deploy/metald/go.mod b/go/deploy/metald/go.mod index 0565a2bc4a..cc764a708a 100644 --- a/go/deploy/metald/go.mod +++ b/go/deploy/metald/go.mod @@ -4,14 +4,13 @@ go 1.24.4 require ( connectrpc.com/connect v1.18.1 + github.com/docker/docker v28.2.2+incompatible + github.com/docker/go-connections v0.5.0 github.com/firecracker-microvm/firecracker-go-sdk v1.0.0 github.com/mattn/go-sqlite3 v1.14.28 github.com/prometheus/client_golang v1.22.0 github.com/stretchr/testify v1.10.0 github.com/unkeyed/unkey/go v0.0.0-00010101000000-000000000000 - github.com/unkeyed/unkey/go/deploy/assetmanagerd v0.0.0-20250709084132-0dd195e4c51b - github.com/unkeyed/unkey/go/deploy/billaged v0.0.0-20250709084132-0dd195e4c51b - github.com/unkeyed/unkey/go/deploy/builderd v0.0.0-20250709084132-0dd195e4c51b github.com/unkeyed/unkey/go/deploy/pkg/health v0.0.0-00010101000000-000000000000 github.com/unkeyed/unkey/go/deploy/pkg/observability/interceptors v0.0.0-00010101000000-000000000000 github.com/unkeyed/unkey/go/deploy/pkg/tls v0.0.0-00010101000000-000000000000 @@ -38,10 +37,15 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v5 v5.0.2 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/containerd/errdefs v0.3.0 // indirect + github.com/containerd/errdefs/pkg v0.3.0 // indirect github.com/containerd/fifo v1.1.0 // indirect + github.com/containerd/log v0.1.0 // indirect github.com/containernetworking/cni v1.3.0 // indirect github.com/containernetworking/plugins v1.7.1 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/distribution/reference v0.6.0 // indirect + github.com/docker/go-units v0.5.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-jose/go-jose/v4 v4.0.5 // indirect github.com/go-logr/logr v1.4.3 // indirect @@ -56,6 +60,7 @@ require ( github.com/go-openapi/strfmt v0.23.0 // indirect github.com/go-openapi/swag v0.23.1 // indirect github.com/go-openapi/validate v0.24.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect github.com/google/uuid v1.6.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect @@ -63,9 +68,15 @@ require ( github.com/josharian/intern v1.0.0 // indirect github.com/mailru/easyjson v0.9.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/moby/sys/atomicwriter v0.1.0 // indirect + github.com/morikuni/aec v1.0.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/oklog/ulid v1.3.1 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.1 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.65.0 // indirect @@ -82,6 +93,7 @@ require ( golang.org/x/crypto v0.39.0 // indirect golang.org/x/sync v0.15.0 // indirect golang.org/x/text v0.26.0 // indirect + golang.org/x/time v0.12.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 // indirect google.golang.org/grpc v1.73.0 // indirect diff --git a/go/deploy/metald/go.sum b/go/deploy/metald/go.sum index 2611c095bb..30b310dd1d 100644 --- a/go/deploy/metald/go.sum +++ b/go/deploy/metald/go.sum @@ -26,6 +26,8 @@ connectrpc.com/connect v1.18.1/go.mod h1:0292hj1rnx8oFrStN7cB4jjVBeqs+Yx5yDIC2pr dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-autorest v10.8.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw= @@ -149,6 +151,10 @@ github.com/containerd/continuity v0.0.0-20200710164510-efbc4488d8fe/go.mod h1:cE github.com/containerd/continuity v0.0.0-20201208142359-180525291bb7/go.mod h1:kR3BEg7bDFaEddKm54WSmrol1fKWDU1nKYkgrcgZT7Y= github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ= github.com/containerd/continuity v0.1.0/go.mod h1:ICJu0PwR54nI0yPEnJ6jcS+J7CZAUXrLh8lPo2knzsM= +github.com/containerd/errdefs v0.3.0 h1:FSZgGOeK4yuT/+DnF07/Olde/q4KBoMsaamhXxIMDp4= +github.com/containerd/errdefs v0.3.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= +github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= github.com/containerd/fifo v0.0.0-20200410184934-f15a3290365b/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0= @@ -168,6 +174,8 @@ github.com/containerd/imgcrypt v1.0.1/go.mod h1:mdd8cEPW7TPgNG4FpuP3sGBiQ7Yi/zak github.com/containerd/imgcrypt v1.0.4-0.20210301171431-0ae5c75f59ba/go.mod h1:6TNsg0ctmizkrOgXRNQjAPFWpMYRWuiB6dSF4Pfa5SA= github.com/containerd/imgcrypt v1.1.1-0.20210312161619-7ed62a527887/go.mod h1:5AZJNI6sLHJljKuI9IHnw1pWqo/F0nGDOuR9zgTs7ow= github.com/containerd/imgcrypt v1.1.1/go.mod h1:xpLnwiQmEUJPvQoAapeb2SNCxz7Xr6PJrXQb0Dpc4ms= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/containerd/nri v0.0.0-20201007170849-eb1350a75164/go.mod h1:+2wGSDGFYfE5+So4M5syatU0N0f0LbWpuqyMi4/BE8c= github.com/containerd/nri v0.0.0-20210316161719-dbaa18c31c14/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= github.com/containerd/nri v0.1.0/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= @@ -232,15 +240,23 @@ github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8l github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY= github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/docker v28.2.2+incompatible h1:CjwRSksz8Yo4+RmQ339Dp/D2tGO5JxwYeqtMOEe0LDw= +github.com/docker/docker v28.2.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= +github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-events v0.0.0-20170721190031-9461782956ad/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= github.com/docker/go-metrics v0.0.0-20180209012529-399ea8c73916/go.mod h1:/u0gXw0Gay3ceNrsHubL3BtdOL2fHf93USgMTe0W5dI= github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= @@ -371,6 +387,7 @@ github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zV github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -544,16 +561,26 @@ github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= +github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw= +github.com/moby/sys/atomicwriter v0.1.0/go.mod h1:Ul8oqv2ZMNHOceF643P6FKPXeCmYtlQMvpizfsSoaWs= github.com/moby/sys/mountinfo v0.4.0/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= +github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= +github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= github.com/moby/sys/symlink v0.1.0/go.mod h1:GGDODQmbFOjFsXvfLVn3+ZRxkch54RkSiGqsZeMYowQ= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= +github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ= +github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= @@ -592,9 +619,12 @@ github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1 github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0-rc1.0.20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.0/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runc v1.0.0-rc8.0.20190926000215-3e425f80a8c9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= @@ -617,6 +647,7 @@ github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= @@ -1014,6 +1045,8 @@ golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= +golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1169,9 +1202,12 @@ gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= +gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= +gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/go/deploy/metald/internal/assetmanager/client.go b/go/deploy/metald/internal/assetmanager/client.go index 29d8009d7d..705158af9b 100644 --- a/go/deploy/metald/internal/assetmanager/client.go +++ b/go/deploy/metald/internal/assetmanager/client.go @@ -9,8 +9,8 @@ import ( "time" "connectrpc.com/connect" - assetv1 "github.com/unkeyed/unkey/go/deploy/assetmanagerd/gen/asset/v1" - "github.com/unkeyed/unkey/go/deploy/assetmanagerd/gen/asset/v1/assetv1connect" + assetv1 "github.com/unkeyed/unkey/go/gen/proto/deploy/assetmanagerd/v1" + "github.com/unkeyed/unkey/go/gen/proto/deploy/assetmanagerd/v1/assetmanagerdv1connect" "github.com/unkeyed/unkey/go/deploy/metald/internal/config" "github.com/unkeyed/unkey/go/deploy/metald/internal/observability" "github.com/unkeyed/unkey/go/deploy/pkg/observability/interceptors" @@ -37,7 +37,7 @@ type Client interface { // client implements the Client interface type client struct { - assetClient assetv1connect.AssetManagerServiceClient + assetClient assetmanagerdv1connect.AssetManagerServiceClient logger *slog.Logger } @@ -68,7 +68,7 @@ func NewClient(cfg *config.AssetManagerConfig, logger *slog.Logger) (Client, err interceptorList = append(interceptorList, connect.Interceptor(interceptor)) } - assetClient := assetv1connect.NewAssetManagerServiceClient( + assetClient := assetmanagerdv1connect.NewAssetManagerServiceClient( httpClient, cfg.Endpoint, connect.WithInterceptors(interceptorList...), @@ -101,7 +101,7 @@ func NewClientWithHTTP(cfg *config.AssetManagerConfig, logger *slog.Logger, http interceptorList = append(interceptorList, connect.Interceptor(interceptor)) } - assetClient := assetv1connect.NewAssetManagerServiceClient( + assetClient := assetmanagerdv1connect.NewAssetManagerServiceClient( httpClient, cfg.Endpoint, connect.WithInterceptors(interceptorList...), diff --git a/go/deploy/metald/internal/assetmanager/client_test.go b/go/deploy/metald/internal/assetmanager/client_test.go index 22467583bd..4c30016d9e 100644 --- a/go/deploy/metald/internal/assetmanager/client_test.go +++ b/go/deploy/metald/internal/assetmanager/client_test.go @@ -5,7 +5,7 @@ import ( "log/slog" "testing" - assetv1 "github.com/unkeyed/unkey/go/deploy/assetmanagerd/gen/asset/v1" + assetv1 "github.com/unkeyed/unkey/go/gen/proto/deploy/assetmanagerd/v1" "github.com/unkeyed/unkey/go/deploy/metald/internal/config" ) diff --git a/go/deploy/metald/internal/backend/docker/client.go b/go/deploy/metald/internal/backend/docker/client.go index e77483325b..7e03174298 100644 --- a/go/deploy/metald/internal/backend/docker/client.go +++ b/go/deploy/metald/internal/backend/docker/client.go @@ -11,16 +11,13 @@ import ( "sync" "time" - "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" - "github.com/docker/docker/api/types/network" "github.com/docker/docker/client" "github.com/docker/go-connections/nat" backendtypes "github.com/unkeyed/unkey/go/deploy/metald/internal/backend/types" metaldv1 "github.com/unkeyed/unkey/go/gen/proto/metal/vmprovisioner/v1" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/codes" "go.opentelemetry.io/otel/metric" "go.opentelemetry.io/otel/trace" ) @@ -226,7 +223,7 @@ func (d *DockerBackend) BootVM(ctx context.Context, vmID string) error { ) // Start container - if err := d.dockerClient.ContainerStart(ctx, vm.ContainerID, types.ContainerStartOptions{}); err != nil { + if err := d.dockerClient.ContainerStart(ctx, vm.ContainerID, container.StartOptions{}); err != nil { span.RecordError(err) d.vmErrorCounter.Add(ctx, 1, metric.WithAttributes( attribute.String("operation", "boot"), @@ -298,7 +295,7 @@ func (d *DockerBackend) DeleteVM(ctx context.Context, vmID string) error { ) // Remove container (force remove) - if err := d.dockerClient.ContainerRemove(ctx, vm.ContainerID, types.ContainerRemoveOptions{ + if err := d.dockerClient.ContainerRemove(ctx, vm.ContainerID, container.RemoveOptions{ Force: true, }); err != nil { span.RecordError(err) @@ -355,8 +352,8 @@ func (d *DockerBackend) ShutdownVMWithOptions(ctx context.Context, vmID string, ) // Stop container - timeout := time.Duration(timeoutSeconds) * time.Second - if err := d.dockerClient.ContainerStop(ctx, vm.ContainerID, &timeout); err != nil { + timeoutInt := int(timeoutSeconds) + if err := d.dockerClient.ContainerStop(ctx, vm.ContainerID, container.StopOptions{Timeout: &timeoutInt}); err != nil { span.RecordError(err) return fmt.Errorf("failed to stop container: %w", err) } @@ -541,7 +538,7 @@ func (d *DockerBackend) GetVMMetrics(ctx context.Context, vmID string) (*backend defer stats.Body.Close() // Parse stats - var dockerStats types.StatsJSON + var dockerStats container.StatsResponse if err := json.NewDecoder(stats.Body).Decode(&dockerStats); err != nil { span.RecordError(err) return nil, fmt.Errorf("failed to decode container stats: %w", err) diff --git a/go/deploy/metald/internal/backend/docker/metrics.go b/go/deploy/metald/internal/backend/docker/metrics.go index b2d78079a0..24f81cd937 100644 --- a/go/deploy/metald/internal/backend/docker/metrics.go +++ b/go/deploy/metald/internal/backend/docker/metrics.go @@ -7,7 +7,7 @@ import ( "log/slog" "time" - "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/container" "github.com/docker/docker/client" backendtypes "github.com/unkeyed/unkey/go/deploy/metald/internal/backend/types" "go.opentelemetry.io/otel/attribute" @@ -46,7 +46,7 @@ func (mc *MetricsCollector) CollectMetrics(ctx context.Context, containerID stri defer stats.Body.Close() // Parse stats JSON - var dockerStats types.StatsJSON + var dockerStats container.StatsResponse if err := json.NewDecoder(stats.Body).Decode(&dockerStats); err != nil { span.RecordError(err) return nil, fmt.Errorf("failed to decode container stats: %w", err) @@ -69,7 +69,7 @@ func (mc *MetricsCollector) CollectMetrics(ctx context.Context, containerID stri } // convertDockerStatsToVMMetrics converts Docker stats to VM metrics format -func (mc *MetricsCollector) convertDockerStatsToVMMetrics(stats *types.StatsJSON) *backendtypes.VMMetrics { +func (mc *MetricsCollector) convertDockerStatsToVMMetrics(stats *container.StatsResponse) *backendtypes.VMMetrics { metrics := &backendtypes.VMMetrics{ Timestamp: time.Now(), CpuTimeNanos: 0, @@ -178,7 +178,7 @@ func (mc *MetricsCollector) StreamMetrics(ctx context.Context, containerID strin } // CalculateCPUPercent calculates CPU percentage from Docker stats -func (mc *MetricsCollector) CalculateCPUPercent(current, previous *types.StatsJSON) float64 { +func (mc *MetricsCollector) CalculateCPUPercent(current, previous *container.StatsResponse) float64 { if current == nil || previous == nil { return 0.0 } @@ -203,7 +203,7 @@ func (mc *MetricsCollector) CalculateCPUPercent(current, previous *types.StatsJS } // CalculateMemoryPercent calculates memory percentage from Docker stats -func (mc *MetricsCollector) CalculateMemoryPercent(stats *types.StatsJSON) float64 { +func (mc *MetricsCollector) CalculateMemoryPercent(stats *container.StatsResponse) float64 { if stats == nil || stats.MemoryStats.Limit == 0 { return 0.0 } @@ -216,7 +216,7 @@ func (mc *MetricsCollector) CalculateMemoryPercent(stats *types.StatsJSON) float } // CalculateNetworkIORate calculates network I/O rate from Docker stats -func (mc *MetricsCollector) CalculateNetworkIORate(current, previous *types.StatsJSON, timeDelta time.Duration) (rxRate, txRate float64) { +func (mc *MetricsCollector) CalculateNetworkIORate(current, previous *container.StatsResponse, timeDelta time.Duration) (rxRate, txRate float64) { if current == nil || previous == nil || timeDelta == 0 { return 0.0, 0.0 } @@ -243,7 +243,7 @@ func (mc *MetricsCollector) CalculateNetworkIORate(current, previous *types.Stat } // CalculateBlockIORate calculates block I/O rate from Docker stats -func (mc *MetricsCollector) CalculateBlockIORate(current, previous *types.StatsJSON, timeDelta time.Duration) (readRate, writeRate float64) { +func (mc *MetricsCollector) CalculateBlockIORate(current, previous *container.StatsResponse, timeDelta time.Duration) (readRate, writeRate float64) { if current == nil || previous == nil || timeDelta == 0 { return 0.0, 0.0 } diff --git a/go/deploy/metald/internal/backend/docker/types.go b/go/deploy/metald/internal/backend/docker/types.go index ac7abe965d..2022890f8b 100644 --- a/go/deploy/metald/internal/backend/docker/types.go +++ b/go/deploy/metald/internal/backend/docker/types.go @@ -1,11 +1,9 @@ package docker import ( - "context" "time" - "github.com/docker/docker/api/types/container" - "github.com/docker/docker/api/types/network" + "github.com/docker/go-connections/nat" "github.com/unkeyed/unkey/go/deploy/metald/internal/backend/types" metaldv1 "github.com/unkeyed/unkey/go/gen/proto/metal/vmprovisioner/v1" ) @@ -121,7 +119,7 @@ type ContainerCreateOptions struct { Env []string Labels map[string]string ExposedPorts map[string]struct{} - PortBindings map[string][]container.PortBinding + PortBindings map[string][]nat.PortBinding Memory int64 CPUs float64 WorkingDir string diff --git a/go/deploy/metald/internal/backend/firecracker/automatic_build_test.go b/go/deploy/metald/internal/backend/firecracker/automatic_build_test.go index 1cb1ac8795..effbdc31dc 100644 --- a/go/deploy/metald/internal/backend/firecracker/automatic_build_test.go +++ b/go/deploy/metald/internal/backend/firecracker/automatic_build_test.go @@ -8,7 +8,7 @@ import ( "log/slog" - assetv1 "github.com/unkeyed/unkey/go/deploy/assetmanagerd/gen/asset/v1" + assetv1 "github.com/unkeyed/unkey/go/gen/proto/deploy/assetmanagerd/v1" "github.com/unkeyed/unkey/go/deploy/metald/internal/config" metaldv1 "github.com/unkeyed/unkey/go/gen/proto/metal/vmprovisioner/v1" ) diff --git a/go/deploy/metald/internal/backend/firecracker/sdk_client_v4.go b/go/deploy/metald/internal/backend/firecracker/sdk_client_v4.go index 5f5ca75b1d..8369c283f1 100644 --- a/go/deploy/metald/internal/backend/firecracker/sdk_client_v4.go +++ b/go/deploy/metald/internal/backend/firecracker/sdk_client_v4.go @@ -17,8 +17,8 @@ import ( sdk "github.com/firecracker-microvm/firecracker-go-sdk" "github.com/firecracker-microvm/firecracker-go-sdk/client/models" - assetv1 "github.com/unkeyed/unkey/go/deploy/assetmanagerd/gen/asset/v1" - builderv1 "github.com/unkeyed/unkey/go/deploy/builderd/gen/builder/v1" + assetv1 "github.com/unkeyed/unkey/go/gen/proto/deploy/assetmanagerd/v1" + builderv1 "github.com/unkeyed/unkey/go/gen/proto/deploy/builderd/v1" "github.com/unkeyed/unkey/go/deploy/metald/internal/assetmanager" "github.com/unkeyed/unkey/go/deploy/metald/internal/backend/types" "github.com/unkeyed/unkey/go/deploy/metald/internal/config" diff --git a/go/deploy/metald/internal/backend/firecracker/sdk_client_v4_test.go b/go/deploy/metald/internal/backend/firecracker/sdk_client_v4_test.go index 5447bab0b7..f5dbaf7ae3 100644 --- a/go/deploy/metald/internal/backend/firecracker/sdk_client_v4_test.go +++ b/go/deploy/metald/internal/backend/firecracker/sdk_client_v4_test.go @@ -6,7 +6,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" - assetv1 "github.com/unkeyed/unkey/go/deploy/assetmanagerd/gen/asset/v1" + assetv1 "github.com/unkeyed/unkey/go/gen/proto/deploy/assetmanagerd/v1" metaldv1 "github.com/unkeyed/unkey/go/gen/proto/metal/vmprovisioner/v1" ) diff --git a/go/deploy/metald/internal/backend/types/backend.go b/go/deploy/metald/internal/backend/types/backend.go index bee7c95142..3b6c8675fb 100644 --- a/go/deploy/metald/internal/backend/types/backend.go +++ b/go/deploy/metald/internal/backend/types/backend.go @@ -69,6 +69,7 @@ type BackendType string const ( BackendTypeCloudHypervisor BackendType = "cloudhypervisor" BackendTypeFirecracker BackendType = "firecracker" + BackendTypeDocker BackendType = "docker" ) // VMMetrics contains VM resource usage data for billing diff --git a/go/deploy/metald/internal/billing/client.go b/go/deploy/metald/internal/billing/client.go index 8ee4c77510..93857983e3 100644 --- a/go/deploy/metald/internal/billing/client.go +++ b/go/deploy/metald/internal/billing/client.go @@ -9,8 +9,8 @@ import ( "time" "connectrpc.com/connect" - billingv1 "github.com/unkeyed/unkey/go/deploy/billaged/gen/billing/v1" - "github.com/unkeyed/unkey/go/deploy/billaged/gen/billing/v1/billingv1connect" + billingv1 "github.com/unkeyed/unkey/go/gen/proto/deploy/billaged/v1" + "github.com/unkeyed/unkey/go/gen/proto/deploy/billaged/v1/billagedv1connect" "github.com/unkeyed/unkey/go/deploy/metald/internal/backend/types" "github.com/unkeyed/unkey/go/deploy/metald/internal/observability" "github.com/unkeyed/unkey/go/deploy/pkg/observability/interceptors" @@ -111,7 +111,7 @@ var _ BillingClient = (*MockBillingClient)(nil) type ConnectRPCBillingClient struct { endpoint string logger *slog.Logger - client billingv1connect.BillingServiceClient + client billagedv1connect.BillingServiceClient } func NewConnectRPCBillingClient(endpoint string, logger *slog.Logger) *ConnectRPCBillingClient { @@ -120,7 +120,7 @@ func NewConnectRPCBillingClient(endpoint string, logger *slog.Logger) *ConnectRP } // AIDEV-NOTE: Using debug interceptor for comprehensive error tracking - billingClient := billingv1connect.NewBillingServiceClient( + billingClient := billagedv1connect.NewBillingServiceClient( httpClient, endpoint, connect.WithInterceptors( @@ -151,7 +151,7 @@ func NewConnectRPCBillingClientWithHTTP(endpoint string, logger *slog.Logger, ht interceptorList = append(interceptorList, connect.Interceptor(interceptor)) } - billingClient := billingv1connect.NewBillingServiceClient( + billingClient := billagedv1connect.NewBillingServiceClient( httpClient, endpoint, connect.WithInterceptors(interceptorList...), diff --git a/go/deploy/metald/internal/config/config.go b/go/deploy/metald/internal/config/config.go index 17be2d2bff..b4d8c651e7 100644 --- a/go/deploy/metald/internal/config/config.go +++ b/go/deploy/metald/internal/config/config.go @@ -391,9 +391,9 @@ func LoadConfigWithSocketPathAndLogger(socketPath string, logger *slog.Logger) ( // Validate validates the configuration func (c *Config) Validate() error { - // AIDEV-BUSINESS_RULE: Only Firecracker backend is supported - if c.Backend.Type != types.BackendTypeFirecracker { - return fmt.Errorf("only firecracker backend is supported, got: %s", c.Backend.Type) + // AIDEV-BUSINESS_RULE: Support Firecracker and Docker backends + if c.Backend.Type != types.BackendTypeFirecracker && c.Backend.Type != types.BackendTypeDocker { + return fmt.Errorf("only firecracker and docker backends are supported, got: %s", c.Backend.Type) } // AIDEV-NOTE: Comprehensive unit tests implemented in config_test.go