Skip to content

Commit 110cf5d

Browse files
aaronmondalaaronmondal
committed
Add Kubernetes example
This example starts a fairly complete Kubernetes cluster and showcases perfectly reproducible remote execution via the local remote execution toolchain containers. This example uses a three-layer setup process: 1. The infra layer is a kind cluster with Cilium and MetalLB. This layer is built to be easily swappable with more "production grade" clusters. 2. The operations layer deploys a few standard applications that are not inherently required for NativeLink, but are solid deployments that one would likely want running in a cluster. This includes monitoring and handling image availability. 3. The application layer is a straightforward `kubectl apply -k .` which deploys a NativeLink CAS, Worker and Scheduler. This deployment differs from the Docker Compose setup in that it does not make use of any system paths and doesn't allow visibility "outside" of the node itself. That is, it's a hard requirement that the worker image is self-contained. Storage is fully ephemeral in this example and a `kubectl delete -k .` will destroy the cache for quick iterations and cache testing.
1 parent 22b5cd8 commit 110cf5d

18 files changed

+795
-1
lines changed

‎.bazelrc

+7
Original file line numberDiff line numberDiff line change
@@ -60,3 +60,10 @@ build:windows --enable_runfiles
6060
build:lre --incompatible_enable_cc_toolchain_resolution
6161
build:lre --define=EXECUTOR=remote
6262
build:lre --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1
63+
64+
# Flags for integration tests running on Kubernetes.
65+
66+
build:k8s --config=lre
67+
build:k8s --remote_instance_name=main
68+
build:k8s --remote_cache=grpc://172.20.255.200:50051
69+
build:k8s --remote_executor=grpc://172.20.255.201:50052

‎.github/workflows/lre.yaml

+54
Original file line numberDiff line numberDiff line change
@@ -38,3 +38,57 @@ jobs:
3838
--config=lre \
3939
--verbose_failures \
4040
//local-remote-execution/examples:hello_lre"
41+
42+
remote:
43+
strategy:
44+
fail-fast: false
45+
matrix:
46+
os: [ubuntu-22.04]
47+
name: Remote / ${{ matrix.os }}
48+
runs-on: ${{ matrix.os }}
49+
steps:
50+
- name: Checkout
51+
uses: >- # v4.1.1
52+
actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
53+
54+
- name: Install Nix
55+
uses: >- #v7
56+
DeterminateSystems/nix-installer-action@5620eb4af6b562c53e4d4628c0b6e4f9d9ae8612
57+
58+
- name: Cache Nix derivations
59+
uses: >- # Custom commit, last pinned at 2023-11-17.
60+
DeterminateSystems/magic-nix-cache-action@a04e6275a6bea232cd04fc6f3cbf20d4cb02a3e1
61+
62+
- name: Start Kubernetes cluster
63+
run: >
64+
nix develop --impure --command
65+
bash -c "cd deployment-examples/kubernetes \
66+
&& ./00_infra.sh \
67+
&& ./01_operations.sh \
68+
&& kubectl apply -k ."
69+
70+
- name: Print gateways
71+
run: |
72+
kubectl get gtw
73+
74+
- name: Get gateway IPs
75+
id: gateway-ips
76+
run: |
77+
echo "cache_ip=$(kubectl get gtw cache -o=jsonpath='{.status.addresses[0].value}')" >> "$GITHUB_ENV"
78+
echo "scheduler_ip=$(kubectl get gtw scheduler -o=jsonpath='{.status.addresses[0].value}')" >> "$GITHUB_ENV"
79+
80+
- name: Print gateways again
81+
run: |
82+
echo 'Cache IP: $cache_ip'
83+
echo 'Scheduler IP: $scheduler_ip'
84+
85+
- name: Build hello_lre with LRE toolchain.
86+
run: >
87+
nix develop --impure --command
88+
bash -c "bazel run \
89+
--config=lre \
90+
--remote_instance_name=main \
91+
--remote_cache=grpc://$cache_ip:50051 \
92+
--remote_executor=grpc://$scheduler_ip:50052 \
93+
--verbose_failures \
94+
//local-remote-execution/examples:hello_lre"

‎deployment-examples/kubernetes/00_infra.sh

+127
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
# This script sets up a local development cluster. It's roughly equivalent to
2+
# a managed K8s setup.
3+
4+
# For ease of development and to save disk space we pipe a local container
5+
# registry through to kind.
6+
#
7+
# See https://kind.sigs.k8s.io/docs/user/local-registry/.
8+
9+
reg_name='kind-registry'
10+
reg_port='5001'
11+
if [ "$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)" != 'true' ]; then
12+
docker run \
13+
-d --restart=always -p "127.0.0.1:${reg_port}:5000" --network bridge --name "${reg_name}" \
14+
registry:2
15+
fi
16+
17+
# Start a basic cluster. We use cilium's CNI and eBPF kube-proxy replacement.
18+
19+
cat <<EOF | kind create cluster --config -
20+
---
21+
kind: Cluster
22+
apiVersion: kind.x-k8s.io/v1alpha4
23+
nodes:
24+
- role: control-plane
25+
- role: worker
26+
- role: worker
27+
networking:
28+
disableDefaultCNI: true
29+
kubeProxyMode: none
30+
containerdConfigPatches:
31+
- |-
32+
[plugins."io.containerd.grpc.v1.cri".registry]
33+
config_path = "/etc/containerd/certs.d"
34+
EOF
35+
36+
# Enable the registry on the nodes.
37+
38+
REGISTRY_DIR="/etc/containerd/certs.d/localhost:${reg_port}"
39+
for node in $(kind get nodes); do
40+
docker exec "${node}" mkdir -p "${REGISTRY_DIR}"
41+
cat <<EOF | docker exec -i "${node}" cp /dev/stdin "${REGISTRY_DIR}/hosts.toml"
42+
[host."http://${reg_name}:5000"]
43+
EOF
44+
done
45+
46+
# Connect the registry to the cluster network.
47+
48+
if [ "$(docker inspect -f='{{json .NetworkSettings.Networks.kind}}' "${reg_name}")" = 'null' ]; then
49+
docker network connect "kind" "${reg_name}"
50+
fi
51+
52+
# Advertise the registry location.
53+
54+
cat <<EOF | kubectl apply -f -
55+
apiVersion: v1
56+
kind: ConfigMap
57+
metadata:
58+
name: local-registry-hosting
59+
namespace: kube-public
60+
data:
61+
localRegistryHosting.v1: |
62+
host: "localhost:${reg_port}"
63+
help: "https://kind.sigs.k8s.io/docs/user/local-registry/"
64+
EOF
65+
66+
# Prepare Gateway API CRDs. These MUST be available before we start cilium.
67+
68+
kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.0.0/experimental-install.yaml
69+
70+
kubectl wait --for condition=Established crd/gatewayclasses.gateway.networking.k8s.io
71+
kubectl wait --for condition=Established crd/gateways.gateway.networking.k8s.io
72+
kubectl wait --for condition=Established crd/httproutes.gateway.networking.k8s.io
73+
kubectl wait --for condition=Established crd/tlsroutes.gateway.networking.k8s.io
74+
kubectl wait --for condition=Established crd/grpcroutes.gateway.networking.k8s.io
75+
kubectl wait --for condition=Established crd/referencegrants.gateway.networking.k8s.io
76+
77+
# Start cilium.
78+
79+
helm repo add cilium https://helm.cilium.io
80+
81+
helm upgrade \
82+
--install cilium cilium/cilium \
83+
--version 1.15.0-pre.3 \
84+
--namespace kube-system \
85+
--set k8sServiceHost=kind-control-plane \
86+
--set k8sServicePort=6443 \
87+
--set kubeProxyReplacement=strict \
88+
--set gatewayAPI.enabled=true \
89+
--wait
90+
91+
# Set up MetalLB. Kind's nodes are containers running on the local docker
92+
# network. We reuse that network for LB-IPAM so that LoadBalancers are available
93+
# via "real" local IPs.
94+
95+
KIND_NET_CIDR=$(docker network inspect kind -f '{{(index .IPAM.Config 0).Subnet}}')
96+
METALLB_IP_START=$(echo ${KIND_NET_CIDR} | sed "[email protected]/[email protected]@")
97+
METALLB_IP_END=$(echo ${KIND_NET_CIDR} | sed "[email protected]/[email protected]@")
98+
METALLB_IP_RANGE="${METALLB_IP_START}-${METALLB_IP_END}"
99+
100+
helm install --namespace metallb-system --create-namespace \
101+
--repo https://metallb.github.io/metallb metallb metallb \
102+
--version 0.13.12 \
103+
--wait
104+
105+
cat <<EOF | kubectl apply -f -
106+
---
107+
apiVersion: metallb.io/v1beta1
108+
kind: L2Advertisement
109+
metadata:
110+
name: l2-ip
111+
namespace: metallb-system
112+
spec:
113+
ipAddressPools:
114+
- default-pool
115+
---
116+
apiVersion: metallb.io/v1beta1
117+
kind: IPAddressPool
118+
metadata:
119+
name: default-pool
120+
namespace: metallb-system
121+
spec:
122+
addresses:
123+
- ${METALLB_IP_RANGE}
124+
EOF
125+
126+
# At this point we have a similar setup to the one that we'd get with a cloud
127+
# provider. Move on to `01_operations.sh` for the cluster setup.

‎deployment-examples/kubernetes/01_operations.sh

+26
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
# This script configures a cluster with a few standard deployments.
2+
3+
# TODO(aaronmondal): Add Grafana, OpenTelemetry and the various other standard
4+
# deployments one would expect in a cluster.
5+
6+
kubectl apply -f gateway.yaml
7+
8+
IMAGE_TAG=$(nix eval .#image.imageTag --raw)
9+
10+
$(nix build .#image --print-build-logs --verbose) \
11+
&& ./result \
12+
| skopeo \
13+
copy \
14+
--dest-tls-verify=false \
15+
docker-archive:/dev/stdin \
16+
docker://localhost:5001/native-link:local
17+
18+
IMAGE_TAG=$(nix eval .#lre.imageTag --raw)
19+
20+
$(nix build .#lre --print-build-logs --verbose) \
21+
&& ./result \
22+
| skopeo \
23+
copy \
24+
--dest-tls-verify=false \
25+
docker-archive:/dev/stdin \
26+
docker://localhost:5001/native-link-toolchain:local

‎deployment-examples/kubernetes/README.md

+39
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
# Kubernetes example
2+
3+
This deployment sets up a 3-container deployment with separate CAS, scheduler
4+
and worker. Don't use this example deployment in production. It's insecure.
5+
6+
In this example we're using `kind` to set up the cluster and `cilium` with
7+
`metallb` to provide a `LoadBalancer` and `GatewayController`.
8+
9+
First set up a local development cluster:
10+
11+
```
12+
./00_infra.sh
13+
```
14+
15+
Next start a few standard deployments. This part also builds the remote
16+
execution containers and makes them available to the cluster:
17+
18+
```
19+
./01_operations.sh
20+
```
21+
22+
Finally deploy NativeLink:
23+
24+
```
25+
kubectl apply -k .
26+
```
27+
28+
Now you can use the `k8s` configuration for Bazel to use the exposed remote
29+
cache and executor:
30+
31+
```
32+
bazel test --config=k8s //:dummy_test
33+
```
34+
35+
When you're done testing, delete the cluster:
36+
37+
```
38+
kind delete cluster
39+
```

‎deployment-examples/kubernetes/cas.json

+100
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
// This configuration will place objects in various folders in
2+
// `~/.cache/native-link`. It will store all data on disk and
3+
// allows for restarts of the underlying service. It is optimized
4+
// so objects are compressed, deduplicated and uses some in-memory
5+
// optimizations for certain hot paths.
6+
{
7+
"stores": {
8+
"CAS_MAIN_STORE": {
9+
"verify": {
10+
"backend": {
11+
"compression": {
12+
"compression_algorithm": {
13+
"LZ4": {}
14+
},
15+
"backend": {
16+
"filesystem": {
17+
"content_path": "~/.cache/native-link/content_path-cas",
18+
"temp_path": "~/.cache/native-link/tmp_path-cas",
19+
"eviction_policy": {
20+
// 10gb.
21+
"max_bytes": 10000000000,
22+
}
23+
}
24+
}
25+
}
26+
},
27+
"verify_size": true,
28+
"verify_hash": true
29+
}
30+
},
31+
"AC_MAIN_STORE": {
32+
"filesystem": {
33+
"content_path": "~/.cache/native-link/content_path-ac",
34+
"temp_path": "~/.cache/native-link/tmp_path-ac",
35+
"eviction_policy": {
36+
// 500mb.
37+
"max_bytes": 500000000,
38+
}
39+
}
40+
}
41+
},
42+
"servers": [{
43+
"listen_address": "0.0.0.0:50051",
44+
"services": {
45+
"cas": {
46+
"main": {
47+
"cas_store": "CAS_MAIN_STORE"
48+
}
49+
},
50+
"ac": {
51+
"main": {
52+
"ac_store": "AC_MAIN_STORE"
53+
}
54+
},
55+
"capabilities": {},
56+
"bytestream": {
57+
"cas_stores": {
58+
"main": "CAS_MAIN_STORE",
59+
},
60+
// According to https://github.com/grpc/grpc.github.io/issues/371 16KiB - 64KiB is optimal.
61+
"max_bytes_per_stream": 64000, // 64kb.
62+
}
63+
}
64+
}, {
65+
// Only publish metrics on a private port.
66+
"listen_address": "0.0.0.0:50061",
67+
"services": {
68+
"prometheus": {
69+
"path": "/metrics"
70+
}
71+
}
72+
},
73+
{
74+
"listen_address": "0.0.0.0:50071",
75+
"tls": {
76+
"cert_file": "/root/example-do-not-use-in-prod-rootca.crt",
77+
"key_file": "/root/example-do-not-use-in-prod-key.pem"
78+
},
79+
"services": {
80+
"cas": {
81+
"main": {
82+
"cas_store": "CAS_MAIN_STORE"
83+
}
84+
},
85+
"ac": {
86+
"main": {
87+
"ac_store": "AC_MAIN_STORE"
88+
}
89+
},
90+
"capabilities": {},
91+
"bytestream": {
92+
"cas_stores": {
93+
"main": "CAS_MAIN_STORE",
94+
},
95+
// According to https://github.com/grpc/grpc.github.io/issues/371 16KiB - 64KiB is optimal.
96+
"max_bytes_per_stream": 64000, // 64kb.
97+
}
98+
}
99+
}]
100+
}

0 commit comments

Comments
 (0)