Skip to content

Commit f547c75

Browse files
aaronmondalaaronmondal
committed
Add Kubernetes example
This example starts a fairly complete Kubernetes cluster and showcases perfectly reproducible remote execution via the local remote execution toolchain containers. This example uses a three-layer setup process: 1. The infra layer is a kind cluster with Cilium and MetalLB. This layer is built to be easily swappable with more "production grade" clusters. 2. The operations layer deploys a few standard applications that are not inherently required for NativeLink, but are solid deployments that one would likely want running in a cluster. This includes monitoring and handling image availability. 3. The application layer is a straightforward `kubectl apply -k .` which deploys a NativeLink CAS, Worker and Scheduler. This deployment differs from the Docker Compose setup in that it does not make use of any system paths and doesn't allow visibility "outside" of the node itself. That is, it's a hard requirement that the worker image is self-contained. Storage is fully ephemeral in this example and a `kubectl delete -k .` will destroy the cache for quick iterations and cache testing.
1 parent 22b5cd8 commit f547c75

18 files changed

+800
-1
lines changed

‎.bazelrc

+7
Original file line numberDiff line numberDiff line change
@@ -60,3 +60,10 @@ build:windows --enable_runfiles
6060
build:lre --incompatible_enable_cc_toolchain_resolution
6161
build:lre --define=EXECUTOR=remote
6262
build:lre --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1
63+
64+
# Flags for integration tests running on Kubernetes.
65+
66+
build:k8s --config=lre
67+
build:k8s --remote_instance_name=main
68+
build:k8s --remote_cache=grpc://172.20.255.200:50051
69+
build:k8s --remote_executor=grpc://172.20.255.201:50052

‎.github/workflows/lre.yaml

+57
Original file line numberDiff line numberDiff line change
@@ -38,3 +38,60 @@ jobs:
3838
--config=lre \
3939
--verbose_failures \
4040
//local-remote-execution/examples:hello_lre"
41+
42+
remote:
43+
strategy:
44+
fail-fast: false
45+
matrix:
46+
os: [ubuntu-22.04]
47+
name: Remote / ${{ matrix.os }}
48+
runs-on: ${{ matrix.os }}
49+
steps:
50+
- name: Checkout
51+
uses: >- # v4.1.1
52+
actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
53+
54+
- name: Install Nix
55+
uses: >- #v7
56+
DeterminateSystems/nix-installer-action@5620eb4af6b562c53e4d4628c0b6e4f9d9ae8612
57+
58+
- name: Cache Nix derivations
59+
uses: >- # Custom commit, last pinned at 2023-11-17.
60+
DeterminateSystems/magic-nix-cache-action@a04e6275a6bea232cd04fc6f3cbf20d4cb02a3e1
61+
62+
- name: Start Kubernetes cluster
63+
run: >
64+
nix develop --impure --command
65+
bash -c "cd deployment-examples/kubernetes \
66+
&& ./00_infra.sh \
67+
&& ./01_operations.sh \
68+
&& kubectl apply -k ."
69+
70+
- name: Get gateway IPs
71+
id: gateway-ips
72+
run: |
73+
echo "cache_ip=$(kubectl get gtw cache -o=jsonpath='{.status.addresses[0].value}')" >> "$GITHUB_ENV"
74+
echo "scheduler_ip=$(kubectl get gtw scheduler -o=jsonpath='{.status.addresses[0].value}')" >> "$GITHUB_ENV"
75+
76+
77+
- name: debug cluster
78+
run: |
79+
kubectl get svc -A
80+
kubectl get pod -A
81+
kubectl get svc -A
82+
kubectl get deployments -A
83+
kubectl describe gtw
84+
kubectl logs -l app=native-link-cas
85+
kubectl logs -l app=native-link-worker
86+
kubectl logs -l app=native-link-scheduler
87+
88+
- name: Build hello_lre with LRE toolchain.
89+
run: >
90+
nix develop --impure --command
91+
bash -c "bazel run \
92+
--config=lre \
93+
--remote_instance_name=main \
94+
--remote_cache=grpc://$cache_ip:50051 \
95+
--remote_executor=grpc://$scheduler_ip:50052 \
96+
--verbose_failures \
97+
//local-remote-execution/examples:hello_lre"

‎deployment-examples/kubernetes/00_infra.sh

+127
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
# This script sets up a local development cluster. It's roughly equivalent to
2+
# a managed K8s setup.
3+
4+
# For ease of development and to save disk space we pipe a local container
5+
# registry through to kind.
6+
#
7+
# See https://kind.sigs.k8s.io/docs/user/local-registry/.
8+
9+
reg_name='kind-registry'
10+
reg_port='5001'
11+
if [ "$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)" != 'true' ]; then
12+
docker run \
13+
-d --restart=always -p "127.0.0.1:${reg_port}:5000" --network bridge --name "${reg_name}" \
14+
registry:2
15+
fi
16+
17+
# Start a basic cluster. We use cilium's CNI and eBPF kube-proxy replacement.
18+
19+
cat <<EOF | kind create cluster --config -
20+
---
21+
kind: Cluster
22+
apiVersion: kind.x-k8s.io/v1alpha4
23+
nodes:
24+
- role: control-plane
25+
- role: worker
26+
- role: worker
27+
networking:
28+
disableDefaultCNI: true
29+
kubeProxyMode: none
30+
containerdConfigPatches:
31+
- |-
32+
[plugins."io.containerd.grpc.v1.cri".registry]
33+
config_path = "/etc/containerd/certs.d"
34+
EOF
35+
36+
# Enable the registry on the nodes.
37+
38+
REGISTRY_DIR="/etc/containerd/certs.d/localhost:${reg_port}"
39+
for node in $(kind get nodes); do
40+
docker exec "${node}" mkdir -p "${REGISTRY_DIR}"
41+
cat <<EOF | docker exec -i "${node}" cp /dev/stdin "${REGISTRY_DIR}/hosts.toml"
42+
[host."http://${reg_name}:5000"]
43+
EOF
44+
done
45+
46+
# Connect the registry to the cluster network.
47+
48+
if [ "$(docker inspect -f='{{json .NetworkSettings.Networks.kind}}' "${reg_name}")" = 'null' ]; then
49+
docker network connect "kind" "${reg_name}"
50+
fi
51+
52+
# Advertise the registry location.
53+
54+
cat <<EOF | kubectl apply -f -
55+
apiVersion: v1
56+
kind: ConfigMap
57+
metadata:
58+
name: local-registry-hosting
59+
namespace: kube-public
60+
data:
61+
localRegistryHosting.v1: |
62+
host: "localhost:${reg_port}"
63+
help: "https://kind.sigs.k8s.io/docs/user/local-registry/"
64+
EOF
65+
66+
# Prepare Gateway API CRDs. These MUST be available before we start cilium.
67+
68+
kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.0.0/experimental-install.yaml
69+
70+
kubectl wait --for condition=Established crd/gatewayclasses.gateway.networking.k8s.io
71+
kubectl wait --for condition=Established crd/gateways.gateway.networking.k8s.io
72+
kubectl wait --for condition=Established crd/httproutes.gateway.networking.k8s.io
73+
kubectl wait --for condition=Established crd/tlsroutes.gateway.networking.k8s.io
74+
kubectl wait --for condition=Established crd/grpcroutes.gateway.networking.k8s.io
75+
kubectl wait --for condition=Established crd/referencegrants.gateway.networking.k8s.io
76+
77+
# Start cilium.
78+
79+
helm repo add cilium https://helm.cilium.io
80+
81+
helm upgrade \
82+
--install cilium cilium/cilium \
83+
--version 1.15.0-pre.3 \
84+
--namespace kube-system \
85+
--set k8sServiceHost=kind-control-plane \
86+
--set k8sServicePort=6443 \
87+
--set kubeProxyReplacement=strict \
88+
--set gatewayAPI.enabled=true \
89+
--wait
90+
91+
# Set up MetalLB. Kind's nodes are containers running on the local docker
92+
# network. We reuse that network for LB-IPAM so that LoadBalancers are available
93+
# via "real" local IPs.
94+
95+
KIND_NET_CIDR=$(docker network inspect kind -f '{{(index .IPAM.Config 0).Subnet}}')
96+
METALLB_IP_START=$(echo ${KIND_NET_CIDR} | sed "[email protected]/[email protected]@")
97+
METALLB_IP_END=$(echo ${KIND_NET_CIDR} | sed "[email protected]/[email protected]@")
98+
METALLB_IP_RANGE="${METALLB_IP_START}-${METALLB_IP_END}"
99+
100+
helm install --namespace metallb-system --create-namespace \
101+
--repo https://metallb.github.io/metallb metallb metallb \
102+
--version 0.13.12 \
103+
--wait
104+
105+
cat <<EOF | kubectl apply -f -
106+
---
107+
apiVersion: metallb.io/v1beta1
108+
kind: L2Advertisement
109+
metadata:
110+
name: l2-ip
111+
namespace: metallb-system
112+
spec:
113+
ipAddressPools:
114+
- default-pool
115+
---
116+
apiVersion: metallb.io/v1beta1
117+
kind: IPAddressPool
118+
metadata:
119+
name: default-pool
120+
namespace: metallb-system
121+
spec:
122+
addresses:
123+
- ${METALLB_IP_RANGE}
124+
EOF
125+
126+
# At this point we have a similar setup to the one that we'd get with a cloud
127+
# provider. Move on to `01_operations.sh` for the cluster setup.

‎deployment-examples/kubernetes/01_operations.sh

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
# This script configures a cluster with a few standard deployments.
2+
3+
# TODO(aaronmondal): Add Grafana, OpenTelemetry and the various other standard
4+
# deployments one would expect in a cluster.
5+
6+
kubectl apply -f gateway.yaml
7+
8+
IMAGE_TAG=$(nix eval .#image.imageTag --raw)
9+
10+
$(nix build .#image --print-build-logs --verbose) \
11+
&& ./result \
12+
| skopeo \
13+
copy \
14+
--dest-tls-verify=false \
15+
docker-archive:/dev/stdin \
16+
docker://localhost:5001/native-link:local
17+
18+
IMAGE_TAG=$(nix eval .#lre.imageTag --raw)
19+
20+
echo $IMAGE_TAG
21+
22+
$(nix build .#lre --print-build-logs --verbose) \
23+
&& ./result \
24+
| skopeo \
25+
copy \
26+
--dest-tls-verify=false \
27+
docker-archive:/dev/stdin \
28+
docker://localhost:5001/native-link-toolchain:local

‎deployment-examples/kubernetes/README.md

+39
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
# Kubernetes example
2+
3+
This deployment sets up a 3-container deployment with separate CAS, scheduler
4+
and worker. Don't use this example deployment in production. It's insecure.
5+
6+
In this example we're using `kind` to set up the cluster and `cilium` with
7+
`metallb` to provide a `LoadBalancer` and `GatewayController`.
8+
9+
First set up a local development cluster:
10+
11+
```
12+
./00_infra.sh
13+
```
14+
15+
Next start a few standard deployments. This part also builds the remote
16+
execution containers and makes them available to the cluster:
17+
18+
```
19+
./01_operations.sh
20+
```
21+
22+
Finally deploy NativeLink:
23+
24+
```
25+
kubectl apply -k .
26+
```
27+
28+
Now you can use the `k8s` configuration for Bazel to use the exposed remote
29+
cache and executor:
30+
31+
```
32+
bazel test --config=k8s //:dummy_test
33+
```
34+
35+
When you're done testing, delete the cluster:
36+
37+
```
38+
kind delete cluster
39+
```

‎deployment-examples/kubernetes/cas.json

+100
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
// This configuration will place objects in various folders in
2+
// `~/.cache/native-link`. It will store all data on disk and
3+
// allows for restarts of the underlying service. It is optimized
4+
// so objects are compressed, deduplicated and uses some in-memory
5+
// optimizations for certain hot paths.
6+
{
7+
"stores": {
8+
"CAS_MAIN_STORE": {
9+
"verify": {
10+
"backend": {
11+
"compression": {
12+
"compression_algorithm": {
13+
"LZ4": {}
14+
},
15+
"backend": {
16+
"filesystem": {
17+
"content_path": "~/.cache/native-link/content_path-cas",
18+
"temp_path": "~/.cache/native-link/tmp_path-cas",
19+
"eviction_policy": {
20+
// 10gb.
21+
"max_bytes": 10000000000,
22+
}
23+
}
24+
}
25+
}
26+
},
27+
"verify_size": true,
28+
"verify_hash": true
29+
}
30+
},
31+
"AC_MAIN_STORE": {
32+
"filesystem": {
33+
"content_path": "~/.cache/native-link/content_path-ac",
34+
"temp_path": "~/.cache/native-link/tmp_path-ac",
35+
"eviction_policy": {
36+
// 500mb.
37+
"max_bytes": 500000000,
38+
}
39+
}
40+
}
41+
},
42+
"servers": [{
43+
"listen_address": "0.0.0.0:50051",
44+
"services": {
45+
"cas": {
46+
"main": {
47+
"cas_store": "CAS_MAIN_STORE"
48+
}
49+
},
50+
"ac": {
51+
"main": {
52+
"ac_store": "AC_MAIN_STORE"
53+
}
54+
},
55+
"capabilities": {},
56+
"bytestream": {
57+
"cas_stores": {
58+
"main": "CAS_MAIN_STORE",
59+
},
60+
// According to https://github.com/grpc/grpc.github.io/issues/371 16KiB - 64KiB is optimal.
61+
"max_bytes_per_stream": 64000, // 64kb.
62+
}
63+
}
64+
}, {
65+
// Only publish metrics on a private port.
66+
"listen_address": "0.0.0.0:50061",
67+
"services": {
68+
"prometheus": {
69+
"path": "/metrics"
70+
}
71+
}
72+
},
73+
{
74+
"listen_address": "0.0.0.0:50071",
75+
"tls": {
76+
"cert_file": "/root/example-do-not-use-in-prod-rootca.crt",
77+
"key_file": "/root/example-do-not-use-in-prod-key.pem"
78+
},
79+
"services": {
80+
"cas": {
81+
"main": {
82+
"cas_store": "CAS_MAIN_STORE"
83+
}
84+
},
85+
"ac": {
86+
"main": {
87+
"ac_store": "AC_MAIN_STORE"
88+
}
89+
},
90+
"capabilities": {},
91+
"bytestream": {
92+
"cas_stores": {
93+
"main": "CAS_MAIN_STORE",
94+
},
95+
// According to https://github.com/grpc/grpc.github.io/issues/371 16KiB - 64KiB is optimal.
96+
"max_bytes_per_stream": 64000, // 64kb.
97+
}
98+
}
99+
}]
100+
}

0 commit comments

Comments
 (0)