Skip to content

feat(deploy): redesign metald network architecture #3818

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
chronark merged 2 commits into from
Sep 3, 2025
Merged

feat(deploy): redesign metald network architecture #3818

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4bf86d8
feat(deploy): redesign metald network architecture
imeyer Aug 20, 2025
c2cb27f
fix: appease the compiler gods
imeyer Sep 2, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
631 changes: 0 additions & 631 deletions CLAUDE.md
View file
Open in desktop

This file was deleted.

3 changes: 3 additions & 0 deletions go/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,9 @@ generate:
buf generate
go generate ./...

generate-builder:
buf generate --path proto/deploy/builderd

test: test-unit

test-unit: up
Expand Down
45 changes: 0 additions & 45 deletions go/deploy/CLAUDE.md
View file
Open in desktop

This file was deleted.

20 changes: 4 additions & 16 deletions go/deploy/assetmanagerd/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,11 +36,11 @@ GOLINT := golangci-lint
.DEFAULT_GOAL := help

# Targets (alphabetically ordered)
.PHONY: all build check clean create-user deps dev env-example fmt generate help install lint lint-proto run service-logs service-logs-tail service-restart service-start service-status service-stop setup test test-coverage uninstall version vet
.PHONY: all build check clean deps dev env-example fmt generate help install lint lint-proto run service-logs service-logs-tail service-restart service-start service-status service-stop setup test test-coverage uninstall version vet

all: clean generate build ## Clean, generate, and build
all: clean build ## Clean, generate, and build

build: generate deps ## Build the binary
build: deps ## Build the binary
@mkdir -p $(BUILD_DIR)
@$(GOBUILD) $(LDFLAGS) -o $(BUILD_DIR)/$(BINARY_NAME) ./cmd/$(SERVICE_NAME)

Expand All @@ -50,39 +50,27 @@ clean: ## Clean build artifacts
@rm -rf $(BUILD_DIR)
@rm -rf $(GEN_DIR)

create-user: ## Create service user
@sudo useradd -r -s /bin/false -d $(DATA_DIR) -c "$(SERVICE_NAME) service user" $(SERVICE_NAME) 2>/dev/null || true

deps: ## Download and tidy dependencies
@go mod download
@go mod tidy

dev: generate ## Run in development mode
@go run ./cmd/$(SERVICE_NAME)

env-example: ## Show example environment variables
@echo "Example environment variables for $(SERVICE_NAME):"
@cat .env.example 2>/dev/null || echo "Error: .env.example not found"

fmt: ## Format code
@$(GOFMT) -w .

generate: ## Generate protobuf code
@buf generate
@buf lint

help: ## Show this help message
@echo 'Usage: make [target]'
@echo ''
@echo 'Targets:'
@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf " %-20s %s\n", $$1, $$2}' $(MAKEFILE_LIST)

install: build create-user ## Install the service (requires sudo)
install: build ## Install the service (requires sudo)
@sudo systemctl stop $(SERVICE_NAME) 2>/dev/null || true
@sudo mkdir -p $(CONFIG_DIR)
@sudo cp $(BUILD_DIR)/$(BINARY_NAME) $(INSTALL_DIR)/
@sudo chmod +x $(INSTALL_DIR)/$(BINARY_NAME)
@sudo chown $(SERVICE_NAME):$(SERVICE_NAME) $(CONFIG_DIR)
@sudo cp contrib/systemd/$(SERVICE_NAME).service $(SYSTEMD_DIR)/
@sudo systemctl daemon-reload
@sudo systemctl enable $(SERVICE_NAME) >/dev/null 2>&1
Expand Down
3 changes: 2 additions & 1 deletion go/deploy/assetmanagerd/contrib/systemd/assetmanagerd.service
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
[Unit]
Description=AssetManagerd VM Asset Management Service
Documentation=https://github.com/unkeyed/unkey/go/deploy/assetmanagerd
After=network.target
After=network.target spire-agent.service
Wants=network.target
Requires=spire-agent.service

[Service]
Type=simple
Expand Down
2 changes: 1 addition & 1 deletion go/deploy/assetmanagerd/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ require (
google.golang.org/genproto/googleapis/api v0.0.0-20250707201910-8d1bb00bc6a7 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 // indirect
google.golang.org/grpc v1.73.0 // indirect
google.golang.org/protobuf v1.36.6 // indirect
google.golang.org/protobuf v1.36.8 // indirect
Copy link
Copy Markdown
Contributor

@coderabbitai coderabbitai bot Sep 3, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

💡 Verification agent

🧩 Analysis chain

Proto bump looks fine; verify repo-wide alignment and regen if needed.

Confirm all modules now resolve google.golang.org/protobuf v1.36.8 and regenerate any generated protobuf code if you vendor stubs or pin protoc-gen-go. Run go mod tidy afterwards.

🏁 Script executed:

#!/bin/bash
# Check protobuf version across all modules
fd go.mod | xargs -I{} bash -lc 'echo "== {} =="; rg -n "google.golang.org/protobuf" {} || true'
# Inspect effective deps in this module
( cd go/deploy/assetmanagerd && go list -m all | rg "^google.golang.org/(grpc|protobuf|genproto)" )
# Optional: locate protoc-gen-go usage to decide if regen is needed
rg -nP "(protoc|buf).*protoc-gen-go" -g '!**/node_modules/**' -C2

Length of output: 3027

Align google.golang.org/protobuf to v1.36.8 repository-wide
Bump protobuf to v1.36.8 in the following go.mod files (currently on older versions):

  • apps/agent/go.mod (v1.36.5)
  • apps/chproxy/go.mod (v1.36.6)
  • go/deploy/billaged/go.mod (v1.36.6)
  • go/deploy/metald/go.mod (v1.36.6)
  • go/deploy/pkg/observability/interceptors/go.mod (v1.36.6)
  • go/deploy/pkg/spiffe/go.mod (v1.36.5)
  • go/deploy/pkg/telemetry/go.mod (v1.36.6)
  • go/deploy/pkg/tls/go.mod (v1.36.5)

After aligning versions, regenerate any vendored protobuf stubs or run your protoc-gen-go pipeline (none found via rg search) and run go mod tidy in each module.

)

replace github.com/unkeyed/unkey/go/deploy/pkg/tls => ../pkg/tls
Expand Down
4 changes: 2 additions & 2 deletions go/deploy/assetmanagerd/go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 h1:
google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A=
google.golang.org/grpc v1.73.0 h1:VIWSmpI2MegBtTuFt5/JWy2oXxtjJ/e89Z70ImfD2ok=
google.golang.org/grpc v1.73.0/go.mod h1:50sbHOUqWoCQGI8V2HQLJM0B+LMlIUjNSZmow7EVBQc=
google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY=
google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY=
google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc=
google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
8 changes: 1 addition & 7 deletions go/deploy/assetmanagerd/internal/service/service.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,11 @@ import (
"time"

"connectrpc.com/connect"
assetv1 "github.com/unkeyed/unkey/go/gen/proto/deploy/assetmanagerd/v1"
"github.com/unkeyed/unkey/go/deploy/assetmanagerd/internal/builderd"
"github.com/unkeyed/unkey/go/deploy/assetmanagerd/internal/config"
"github.com/unkeyed/unkey/go/deploy/assetmanagerd/internal/registry"
"github.com/unkeyed/unkey/go/deploy/assetmanagerd/internal/storage"
"github.com/unkeyed/unkey/go/deploy/pkg/observability/interceptors"
assetv1 "github.com/unkeyed/unkey/go/gen/proto/deploy/assetmanagerd/v1"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
Expand Down Expand Up @@ -1038,11 +1037,6 @@ func (s *Service) QueryAssets(
tenantID := buildOpts.GetTenantId()
customerID := "cli-user" // Default fallback

// Try to extract tenant context for proper customer ID
if tenantCtx, ok := interceptors.TenantFromContext(ctx); ok && tenantCtx.CustomerID != "" {
customerID = tenantCtx.CustomerID
}

buildID, err := s.builderdClient.BuildDockerRootfsWithOptions(buildCtx, dockerImage, buildLabels, tenantID, customerID)
if err != nil {
buildSpan.RecordError(err)
Expand Down
3 changes: 2 additions & 1 deletion go/deploy/billaged/contrib/systemd/billaged.service
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
[Unit]
Description=Billaged VM Usage Billing Service
Documentation=https://github.com/unkeyed/unkey/go/deploy/billaged
After=network.target
After=network.target spire-agent.service assetmanagerd.service metald.service
Wants=network.target
Requires=spire-agent.service assetmanagerd.service metald.service
Comment on lines +4 to +6
Copy link
Copy Markdown
Contributor

@coderabbitai coderabbitai bot Sep 3, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

💡 Verification agent

🧩 Analysis chain

Avoid cascading failures; use Wants(+After) for metald and wait for network-online.

Billaged likely doesn’t need a hard dependency on metald. Requires=metald means billaged is stopped if metald stops. Prefer Wants=metald with ordering, and wait for network-online to avoid TLS/mTLS races on boot.

Apply:

-After=network.target spire-agent.service assetmanagerd.service metald.service
-Wants=network.target
-Requires=spire-agent.service assetmanagerd.service metald.service
+After=network-online.target spire-agent.service assetmanagerd.service metald.service
+Wants=network-online.target metald.service
+Requires=spire-agent.service assetmanagerd.service

🏁 Script executed:

#!/bin/bash
set -e

echo "Locating billaged.service file:"
rg -l 'billaged.service'

echo
echo "Printing first 20 lines of billaged.service:"
sed -n '1,20p' go/deploy/billaged/contrib/systemd/billaged.service

echo
echo "Searching for network-online.target usage in all .service files:"
rg -n 'network-online.target' -g '*.service'

Length of output: 2049

🏁 Script executed:

#!/bin/bash
set -e

echo "Locating assetmanagerd.service"
find go -type f -name 'assetmanagerd.service'

echo
echo "Printing first 20 lines of metald.service"
sed -n '1,20p' go/deploy/docker/systemd/metald.service

echo
echo "Printing first 20 lines of spire-agent.service"
sed -n '1,20p' go/deploy/spire/contrib/systemd/spire-agent.service

Length of output: 1733

go/deploy/billaged/contrib/systemd/billaged.service – use Wants=metald.service (+After) and network-online.target

-After=network.target spire-agent.service assetmanagerd.service metald.service
-Wants=network.target
-Requires=spire-agent.service assetmanagerd.service metald.service
+After=network-online.target spire-agent.service assetmanagerd.service metald.service
+Wants=network-online.target metald.service
+Requires=spire-agent.service assetmanagerd.service
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
After=network.target spire-agent.service assetmanagerd.service metald.service
Wants=network.target
Requires=spire-agent.service assetmanagerd.service metald.service
After=network-online.target spire-agent.service assetmanagerd.service metald.service
Wants=network-online.target metald.service
Requires=spire-agent.service assetmanagerd.service
🤖 Prompt for AI Agents 
In go/deploy/billaged/contrib/systemd/billaged.service around lines 4 to 6,
change the unit dependencies so metald is a soft dependency and the service
waits for the network to be online: replace After=network.target with
After=network-online.target spire-agent.service assetmanagerd.service
metald.service; add Wants=network-online.target metald.service (so metald is
pulled in but not a hard require); and remove metald.service from the Requires=
line leaving Requires=spire-agent.service assetmanagerd.service.


[Service]
Type=simple
Expand Down
32 changes: 3 additions & 29 deletions go/deploy/builderd/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,11 +35,11 @@ GOLINT := golangci-lint
.DEFAULT_GOAL := help

# Targets (alphabetically ordered)
.PHONY: all build build-linux check ci clean clean-gen create-user debug deps dev env-example fmt generate help install install-tools lint lint-proto proto-breaking quick-build quick-test release run service-logs service-logs-tail service-restart service-start service-status service-stop setup test test-coverage uninstall version vet
.PHONY: all build build-linux check ci clean clean-gen debug deps dev env-example fmt generate help install install-tools lint lint-proto proto-breaking quick-build quick-test release run service-logs service-logs-tail service-restart service-start service-status service-stop setup test test-coverage uninstall version vet

all: clean generate build ## Clean, generate, and build

build: generate deps ## Build the binary
build: deps ## Build the binary
@mkdir -p $(BUILD_DIR)
@$(GOBUILD) $(LDFLAGS) -o $(BUILD_DIR)/$(BINARY_NAME) ./cmd/$(SERVICE_NAME)

Expand All @@ -56,12 +56,6 @@ clean: ## Clean build artifacts
@rm -rf $(GEN_DIR)
@rm -f coverage.out coverage.html

clean-gen: ## Clean generated protobuf code
@rm -rf $(GEN_DIR)

create-user: ## Create service user
@sudo useradd -r -s /bin/false -d /opt/builderd -c "$(SERVICE_NAME) service user" $(SERVICE_NAME) 2>/dev/null || true

debug: build ## Run with debug logging
@UNKEY_BUILDERD_LOG_LEVEL=debug ./$(BUILD_DIR)/$(BINARY_NAME)

Expand All @@ -72,13 +66,6 @@ deps: ## Download and tidy dependencies
dev: ## Run in development mode
@go run ./cmd/$(SERVICE_NAME)

env-example: ## Show example environment variables
@echo "Example environment variables for $(SERVICE_NAME):"
@echo "UNKEY_BUILDERD_PORT=8082"
@echo "UNKEY_BUILDERD_OTEL_ENABLED=false"
@echo "UNKEY_BUILDERD_STORAGE_BACKEND=local"
@echo "UNKEY_BUILDERD_LOG_LEVEL=info"

fmt: ## Format code
@$(GOFMT) -w .
@which goimports >/dev/null && goimports -w . || echo "goimports not found, install with: go install golang.org/x/tools/cmd/goimports@latest"
Expand All @@ -93,29 +80,17 @@ help: ## Show this help message
@echo 'Targets:'
@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf " %-20s %s\n", $$1, $$2}' $(MAKEFILE_LIST)

install: build create-user ## Install the service (requires sudo)
install: build ## Install the service (requires sudo)
@sudo systemctl stop $(SERVICE_NAME) 2>/dev/null || true
@sudo mkdir -p $(CONFIG_DIR)
@sudo cp $(BUILD_DIR)/$(BINARY_NAME) $(INSTALL_DIR)/
@sudo chmod +x $(INSTALL_DIR)/$(BINARY_NAME)
@sudo chown $(SERVICE_NAME):$(SERVICE_NAME) $(CONFIG_DIR)
@sudo cp contrib/systemd/$(SERVICE_NAME).service $(SYSTEMD_DIR)/
@sudo systemctl daemon-reload
@sudo systemctl enable $(SERVICE_NAME) >/dev/null 2>&1
@sudo systemctl start $(SERVICE_NAME) 2>/dev/null || true
@echo "✓ $(SERVICE_NAME) installed and started"


lint: lint-proto ## Run linter (includes protobuf linting)
@which $(GOLINT) >/dev/null || (echo "golangci-lint not found, install with: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $$(go env GOPATH)/bin v1.54.2" && exit 1)
@$(GOLINT) run --disable=godox

lint-proto: ## Run protobuf linter
@buf lint

proto-breaking: ## Check for breaking protobuf changes
@buf breaking --against .git#branch=main

quick-build: ## Quick build without optimizations
@mkdir -p $(BUILD_DIR)
@$(GOBUILD) -o $(BUILD_DIR)/$(BINARY_NAME) ./cmd/$(SERVICE_NAME)
Expand Down Expand Up @@ -175,4 +150,3 @@ version: ## Show version information

vet: ## Run go vet
@$(GOVET) ./...

2 changes: 1 addition & 1 deletion go/deploy/builderd/client/client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@ import (
"time"

"connectrpc.com/connect"
"github.com/unkeyed/unkey/go/deploy/pkg/tls"
builderv1 "github.com/unkeyed/unkey/go/gen/proto/deploy/builderd/v1"
"github.com/unkeyed/unkey/go/gen/proto/deploy/builderd/v1/builderdv1connect"
"github.com/unkeyed/unkey/go/deploy/pkg/tls"
)

// AIDEV-NOTE: Builderd client with SPIFFE/SPIRE socket integration
Expand Down
Loading