diff --git a/.github/workflows/test-integration.yml b/.github/workflows/test-integration.yml index ca0e3aef46..a22d7fe3c8 100644 --- a/.github/workflows/test-integration.yml +++ b/.github/workflows/test-integration.yml @@ -47,7 +47,11 @@ jobs: # state; the idempotent install-cilium.sh is safe to re-run # after k3s recovers. run: | - curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--flannel-backend=none --disable-network-policy --write-kubeconfig-mode=644" sh - + # --disable=metrics-server: egg doesn't use it, and under Cilium it + # never becomes Ready (its pod can't reach the kubelet on the node + # IP). The dead v1beta1.metrics.k8s.io APIService then wedges all + # namespace deletion, hanging test teardown and CI cleanup. + curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--flannel-backend=none --disable-network-policy --disable=metrics-server --write-kubeconfig-mode=644" sh - export KUBECONFIG=/etc/rancher/k3s/k3s.yaml echo "KUBECONFIG=/etc/rancher/k3s/k3s.yaml" >> "$GITHUB_ENV" # Install Cilium CNI (retry once if k3s becomes unstable mid-rollout) @@ -139,15 +143,11 @@ jobs: - name: Deploy egg to k3s # `make deploy` handles secret creation, image-tag rewrite, host- # path envsubst, and waits on both `gateway` and `orchestrator` - # deployments. Matches the local-dev invocation exactly. - # - # `make k3s-secrets` (a prerequisite of `make deploy`) creates the - # Secret in the `egg-system` namespace, but the namespace itself - # is only created later by `kubectl apply -k k8s/...`. Pre-create - # it here so the secret-create step succeeds. - run: | - kubectl create namespace egg-system --dry-run=client -o yaml | kubectl apply -f - - make deploy + # deployments. Matches the local-dev invocation exactly. The + # `k3s-secrets` prerequisite of `make deploy` applies + # `k8s/base/namespaces.yaml` before creating the Secret, so no + # namespace pre-create is needed in the workflow. + run: make deploy - name: Run integration and security tests env: diff --git a/Makefile b/Makefile index cda214f468..eaf4e106ae 100644 --- a/Makefile +++ b/Makefile @@ -477,9 +477,14 @@ build: sync-venv-if-uv # --flannel-backend=none: Cilium replaces flannel as the CNI dataplane. # --disable-network-policy: Cilium owns NetworkPolicy enforcement; the # k3s-builtin policy controller would otherwise conflict. +# --disable=metrics-server: egg does not use metrics-server. Under Cilium +# its pod cannot reach the kubelet on the node IP, so it never becomes +# Ready; the resulting perpetually-unavailable v1beta1.metrics.k8s.io +# APIService makes the namespace controller's discovery step fail, +# which wedges *all* namespace deletion (stuck Terminating forever). k3s-setup: ## Install k3s with Cilium CNI @echo "Setting up k3s cluster..." - curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--flannel-backend=none --disable-network-policy --write-kubeconfig-mode=644" sh - + curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--flannel-backend=none --disable-network-policy --disable=metrics-server --write-kubeconfig-mode=644" sh - export KUBECONFIG=/etc/rancher/k3s/k3s.yaml && \ scripts/install-cilium.sh && \ echo "Waiting for k3s node to be ready..." && \ @@ -500,6 +505,7 @@ k3s-secrets: ## Create gateway secrets from ~/.config/egg/ @echo "==> Creating gateway-secrets in egg-system namespace..." @echo " (all files under ~/.config/egg/ become keys in the secret)" export KUBECONFIG=$${KUBECONFIG:-/etc/rancher/k3s/k3s.yaml} && \ + kubectl apply -f k8s/base/namespaces.yaml && \ kubectl -n egg-system create secret generic gateway-secrets \ --from-file=$$HOME/.config/egg/ \ --dry-run=client -o yaml | kubectl apply -f - @@ -522,8 +528,7 @@ deploy: k3s-secrets ## Deploy egg to k3s -e "s|egg-gateway:latest|egg-gateway:$(EGG_IMAGE_TAG)|g" \ -e "s|egg-sandbox:latest|egg-sandbox:$(EGG_IMAGE_TAG)|g" | \ kubectl apply -f - && \ - kubectl -n egg-system wait --for=condition=Available deployment/orchestrator --timeout=120s && \ - kubectl -n egg-system wait --for=condition=Available deployment/gateway --timeout=120s + scripts/await-egg-deploy.sh "$(EGG_IMAGE_TAG)" @echo "Deployment complete" redeploy: build k3s-import deploy ## Rebuild, re-import, and redeploy in one step diff --git a/integration_tests/test_deployment_validation_logic.py b/integration_tests/test_deployment_validation_logic.py index 9ad1c96a46..8ab2e56b81 100644 --- a/integration_tests/test_deployment_validation_logic.py +++ b/integration_tests/test_deployment_validation_logic.py @@ -587,6 +587,17 @@ def test_validation_routes_never_leak_secrets_in_error_messages( body={"pipeline_id": "leak test"}, ), ] + # The windowed substring check below needs a secret long enough + # to slice into meaningful windows. The Makefile generates 64 + # hex chars (`openssl rand -hex 32`); a short/non-standard + # lifecycle-secret would otherwise produce empty windows, and + # `"" not in resp.text` is always False — a phantom "leak" + # report for a response that contains no secret at all. + assert len(lifecycle_secret) >= 32, ( + f"lifecycle-secret is only {len(lifecycle_secret)} chars; " + f"expected 64 hex chars. Regenerate it: " + f"openssl rand -hex 32 > ~/.config/egg/lifecycle-secret" + ) for resp in responses: # Full-secret check: 64 hex chars is more than enough # entropy that a false positive is impossible. Catches @@ -597,10 +608,17 @@ def test_validation_routes_never_leak_secrets_in_error_messages( f"{resp.request.url}: {resp.text[:500]}" ) # Also guard against substring leaks (e.g. a bug that - # printed the last 16 hex chars). Use a couple of - # non-overlapping windows. - for start in (0, 16, 32, 48): + # printed the last 16 hex chars). Walk non-overlapping + # 16-char windows across the *actual* secret length — the + # old hardcoded (0, 16, 32, 48) offsets produced empty + # windows for any secret shorter than 64 chars. + for start in range(0, len(lifecycle_secret), 16): window = lifecycle_secret[start : start + 16] + if len(window) < 8: + # Trailing short window — too few chars to be a + # meaningful leak signal (and risks false + # positives); the full-secret check above covers it. + continue assert window not in resp.text, ( f"response body contained a 16-char window of the " f"bearer secret starting at offset {start} — " diff --git a/orchestrator/Dockerfile b/orchestrator/Dockerfile index 56aeb8e90c..aec881048e 100644 --- a/orchestrator/Dockerfile +++ b/orchestrator/Dockerfile @@ -10,15 +10,26 @@ RUN apt-get update && apt-get install -y \ # it `_run_kustomize` falls through both kustomize and `kubectl # kustomize` invocations and the route returns 500 # `kustomize_unavailable` for every call (#2647). Pinned to a known- -# good release; bump deliberately. SHA256 is the published checksum -# for the linux_amd64 tarball (see checksums.txt alongside the -# release); bumping KUSTOMIZE_VERSION requires updating -# KUSTOMIZE_SHA256 in lockstep (#2681). +# good release; bump deliberately. The SHA256s are the published +# checksums for the per-arch tarballs (see checksums.txt alongside the +# release); bumping KUSTOMIZE_VERSION requires updating both in +# lockstep (#2681). Arch is resolved at build time via +# `dpkg --print-architecture` so the image is correct on both amd64 +# and arm64 hosts — matching the arch-aware pattern in the gateway and +# sandbox Dockerfiles (previously hardcoded amd64, which crashed under +# emulation on arm64 dev machines; #2641/#2658). ARG KUSTOMIZE_VERSION=5.6.0 -ARG KUSTOMIZE_SHA256=54e4031ddc4e7fc59e408da29e7c646e8e57b8088c51b84b3df0864f47b5148f -RUN curl -fsSL "https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv${KUSTOMIZE_VERSION}/kustomize_v${KUSTOMIZE_VERSION}_linux_amd64.tar.gz" \ +ARG KUSTOMIZE_SHA256_AMD64=54e4031ddc4e7fc59e408da29e7c646e8e57b8088c51b84b3df0864f47b5148f +ARG KUSTOMIZE_SHA256_ARM64=ad8ab62d4f6d59a8afda0eec4ba2e5cd2f86bf1afeea4b78d06daac945eb0660 +RUN arch="$(dpkg --print-architecture)" \ + && case "$arch" in \ + amd64) sha256="${KUSTOMIZE_SHA256_AMD64}" ;; \ + arm64) sha256="${KUSTOMIZE_SHA256_ARM64}" ;; \ + *) echo "unsupported architecture: $arch" >&2; exit 1 ;; \ + esac \ + && curl -fsSL "https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv${KUSTOMIZE_VERSION}/kustomize_v${KUSTOMIZE_VERSION}_linux_${arch}.tar.gz" \ -o /tmp/kustomize.tar.gz \ - && echo "${KUSTOMIZE_SHA256} /tmp/kustomize.tar.gz" | sha256sum -c - \ + && echo "${sha256} /tmp/kustomize.tar.gz" | sha256sum -c - \ && tar -xzf /tmp/kustomize.tar.gz -C /usr/local/bin kustomize \ && rm /tmp/kustomize.tar.gz \ && chmod +x /usr/local/bin/kustomize diff --git a/scripts/await-egg-deploy.sh b/scripts/await-egg-deploy.sh new file mode 100755 index 0000000000..64ea0d5d00 --- /dev/null +++ b/scripts/await-egg-deploy.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# +# await-egg-deploy.sh - Wait for the egg-system deployments to become +# Available, failing fast with an actionable message when the cause is +# an image tag that was never imported into k3s. +# +# The orchestrator/gateway manifests reference egg-*:, +# where EGG_IMAGE_TAG is `git describe --always` and so changes on every +# commit, pull, rebase, or checkout. `make redeploy` builds, imports, and +# deploys in one invocation so the tag is self-consistent — but a bare +# `make deploy` after HEAD has moved references a tag whose images are +# not in k3s's containerd, and the pods sit in ImagePullBackOff. +# +# Without this guard that surfaces only as a bare 120s `kubectl wait` +# timeout ("error: timed out waiting for the condition"). Here we detect +# the ImagePullBackOff within seconds and point at the fix. +# +set -euo pipefail + +NS="egg-system" +: "${1:?usage: $0 [timeout-seconds]}" +TAG="$1" +TIMEOUT="${2:-180}" +DEPLOYMENTS=(orchestrator gateway) + +# Keep kubectl stderr off of stdout so the success-path jsonpath value in +# $out is strictly equal to the queried field. If a future cluster ever +# emits a stderr warning on a successful call (admission webhook +# deprecation, API-version advisory, token-near-expiry), merging it into +# $out would silently fail the [ "$out" = "True" ] equality and poll +# until timeout. +err_file=$(mktemp) +trap 'rm -f "$err_file"' EXIT + +deadline=$(( $(date +%s) + TIMEOUT )) + +while :; do + # Success: every deployment reports Available=True. + all_available=1 + for d in "${DEPLOYMENTS[@]}"; do + rc=0 + out=$(kubectl -n "$NS" get deployment "$d" \ + -o jsonpath='{.status.conditions[?(@.type=="Available")].status}' 2>"$err_file") || rc=$? + err=$(<"$err_file") + if [ "$rc" -ne 0 ] && ! grep -q 'NotFound' <<<"$err"; then + # Real kubectl error (auth, connection, RBAC) — surface + # immediately rather than polling silently for the full timeout. + # NotFound is the expected "not yet observed" state during early + # rollout and falls through to the not-Available branch below. + echo "ERROR: kubectl get deployment $d failed: $err" >&2 + exit 1 + fi + [ "$out" = "True" ] || all_available=0 + done + if [ "$all_available" -eq 1 ]; then + echo "All egg-system deployments are Available." + exit 0 + fi + + # Fast-fail: a pod can't pull its image. Almost always tag drift — + # HEAD moved since the last build+import, so `make deploy` references + # egg-*:$TAG which was never imported into k3s. + if kubectl -n "$NS" get pods \ + -o jsonpath='{range .items[*]}{range .status.containerStatuses[*]}{.state.waiting.reason}{"\n"}{end}{end}' \ + 2>/dev/null | grep -qE 'ImagePullBackOff|ErrImagePull'; then + echo "ERROR: egg-system pods cannot pull image tag '${TAG}' — it is not in k3s." >&2 + echo " A commit, pull, or rebase since your last build moved EGG_IMAGE_TAG." >&2 + echo " 'make deploy' alone only deploys; run 'make redeploy' to rebuild +" >&2 + echo " re-import + deploy on the current tag." >&2 + exit 1 + fi + + if [ "$(date +%s)" -ge "$deadline" ]; then + echo "ERROR: timed out after ${TIMEOUT}s waiting for egg-system deployments." >&2 + echo " Current pod state:" >&2 + kubectl -n "$NS" get pods >&2 || true + exit 1 + fi + + sleep 3 +done diff --git a/scripts/install-cilium.sh b/scripts/install-cilium.sh index 82605ad76f..de8b287a40 100755 --- a/scripts/install-cilium.sh +++ b/scripts/install-cilium.sh @@ -58,63 +58,117 @@ if kubectl get crd -o name 2>/dev/null | grep -q '\.projectcalico\.org$'; then exit 1 fi -# If Cilium is already installed and ready, exit early. +# If Cilium is already installed and ready, skip the install step but +# still run the post-install verification at the end — that way operators +# can use this script as a config audit on a live cluster without having +# to teardown and reinstall to see whether the deployed cilium-config +# matches the conservative datapath flags this script intends. +SKIP_INSTALL=0 if kubectl get daemonset -n kube-system cilium &>/dev/null; then DESIRED=$(kubectl get daemonset -n kube-system cilium -o jsonpath='{.status.desiredNumberScheduled}') READY=$(kubectl get daemonset -n kube-system cilium -o jsonpath='{.status.numberReady}') if [ "$DESIRED" -gt 0 ] && [ "$DESIRED" = "$READY" ]; then log "Cilium is already installed and all ${READY}/${DESIRED} nodes are ready." - log "To force reinstall, delete the cilium daemonset first." - exit 0 + log "Skipping install; running post-install verification only." + log "(To force reinstall, delete the cilium daemonset first.)" + SKIP_INSTALL=1 else log "Cilium is installed but not fully ready (${READY}/${DESIRED} nodes ready)." log "Re-running install to bring it to ready..." fi fi -# Detect arch -ARCH=$(uname -m) -case "$ARCH" in - aarch64 | arm64) CLI_ARCH="arm64"; CLI_SHA256="$CILIUM_CLI_SHA256_ARM64" ;; - x86_64 | amd64) CLI_ARCH="amd64"; CLI_SHA256="$CILIUM_CLI_SHA256_AMD64" ;; - *) error "Unsupported architecture: $ARCH"; exit 1 ;; -esac +if [ "$SKIP_INSTALL" -eq 0 ]; then + # Detect arch + ARCH=$(uname -m) + case "$ARCH" in + aarch64 | arm64) CLI_ARCH="arm64"; CLI_SHA256="$CILIUM_CLI_SHA256_ARM64" ;; + x86_64 | amd64) CLI_ARCH="amd64"; CLI_SHA256="$CILIUM_CLI_SHA256_AMD64" ;; + *) error "Unsupported architecture: $ARCH"; exit 1 ;; + esac -log "Installing Cilium ${CILIUM_VERSION} via cilium-cli ${CILIUM_CLI_VERSION} (${CLI_ARCH})..." + log "Installing Cilium ${CILIUM_VERSION} via cilium-cli ${CILIUM_CLI_VERSION} (${CLI_ARCH})..." -TMPDIR=$(mktemp -d /tmp/cilium-install.XXXXXX) -trap 'rm -rf "$TMPDIR"' EXIT + TMPDIR=$(mktemp -d /tmp/cilium-install.XXXXXX) + trap 'rm -rf "$TMPDIR"' EXIT -TARBALL="$TMPDIR/cilium-linux-${CLI_ARCH}.tar.gz" -CLI_URL="https://github.com/cilium/cilium-cli/releases/download/${CILIUM_CLI_VERSION}/cilium-linux-${CLI_ARCH}.tar.gz" + TARBALL="$TMPDIR/cilium-linux-${CLI_ARCH}.tar.gz" + CLI_URL="https://github.com/cilium/cilium-cli/releases/download/${CILIUM_CLI_VERSION}/cilium-linux-${CLI_ARCH}.tar.gz" -log "Downloading cilium-cli from ${CLI_URL}..." -if ! curl -fsSL "$CLI_URL" -o "$TARBALL"; then - error "Failed to download cilium-cli" - exit 1 + log "Downloading cilium-cli from ${CLI_URL}..." + if ! curl -fsSL "$CLI_URL" -o "$TARBALL"; then + error "Failed to download cilium-cli" + exit 1 + fi + + log "Verifying cilium-cli checksum..." + ACTUAL_SHA256=$(sha256sum "$TARBALL" | awk '{print $1}') + if [ "$ACTUAL_SHA256" != "$CLI_SHA256" ]; then + error "Checksum mismatch for cilium-cli tarball!" + error " Expected: $CLI_SHA256" + error " Actual: $ACTUAL_SHA256" + error "The downloaded tarball may have been tampered with." + exit 1 + fi + log "Checksum verified." + + tar -xzf "$TARBALL" -C "$TMPDIR" + CILIUM_BIN="$TMPDIR/cilium" + chmod +x "$CILIUM_BIN" + + # Conservative datapath config. kube-proxy replacement, BPF masquerade, + # and BPF host routing each attach eBPF programs to physical devices; on + # hosts where the primary NIC is unusual (e.g. a wireless interface) that + # can blackhole host connectivity entirely. The legacy/iptables datapath + # keeps Cilium's eBPF on cilium_* interfaces and pod veths only, and still + # provides full L3/L4 NetworkPolicy enforcement. + CILIUM_INSTALL_ARGS=( + --version "$CILIUM_VERSION" + --set kubeProxyReplacement=false + --set bpf.masquerade=false + --set bpf.hostLegacyRouting=true + ) + log "Running 'cilium install ${CILIUM_INSTALL_ARGS[*]}'..." + "$CILIUM_BIN" install "${CILIUM_INSTALL_ARGS[@]}" + + log "Waiting for Cilium to be ready (timeout: 300s)..." + "$CILIUM_BIN" status --wait --wait-duration=5m fi -log "Verifying cilium-cli checksum..." -ACTUAL_SHA256=$(sha256sum "$TARBALL" | awk '{print $1}') -if [ "$ACTUAL_SHA256" != "$CLI_SHA256" ]; then - error "Checksum mismatch for cilium-cli tarball!" - error " Expected: $CLI_SHA256" - error " Actual: $ACTUAL_SHA256" - error "The downloaded tarball may have been tampered with." +# Post-install verification: confirm cilium-config matches the conservative +# datapath flags we passed. cilium-cli's auto-detection prints info messages +# like "Cilium will fully replace all functionalities of kube-proxy" even +# when --set kubeProxyReplacement=false is passed (k3s embeds kube-proxy in +# k3s-agent, so cilium-cli sees no kube-proxy DaemonSet and announces it +# will replace it). The --set flag overrides the helm value during chart +# rendering, but the info-message-vs-real-config mismatch is a property we +# should not rely on silently — assert the deployed values match. Runs on +# both the fresh-install and idempotent-skip paths so operators can audit a +# live cluster's config without re-installing. +log "Verifying cilium-config matches expected conservative datapath..." +verify_failed=0 +for kv in \ + 'kube-proxy-replacement:false' \ + 'enable-bpf-masquerade:false' \ + 'enable-host-legacy-routing:true'; do + key="${kv%%:*}" + want="${kv##*:}" + # kubectl jsonpath returns the value directly (empty string if the key + # is absent), which keeps install-cilium.sh free of a jq runtime dep — + # no other script in scripts/ uses jq today. + got=$(kubectl -n kube-system get cm cilium-config -o "jsonpath={.data['$key']}") + if [ "$got" != "$want" ]; then + error "cilium-config[$key] = '$got', expected '$want'" + verify_failed=1 + fi +done +if [ "$verify_failed" -ne 0 ]; then + error "The cilium-cli may have silently overridden a --set flag during install." + error "Run 'kubectl -n kube-system get cm cilium-config -o yaml' to inspect." exit 1 fi -log "Checksum verified." - -tar -xzf "$TARBALL" -C "$TMPDIR" -CILIUM_BIN="$TMPDIR/cilium" -chmod +x "$CILIUM_BIN" - -log "Running 'cilium install --version ${CILIUM_VERSION}'..." -"$CILIUM_BIN" install --version "$CILIUM_VERSION" - -log "Waiting for Cilium to be ready (timeout: 300s)..." -"$CILIUM_BIN" status --wait --wait-duration=5m +log "cilium-config matches expected datapath." # Post-install verification: the host CNI config directory should now # contain Cilium's conflist and nothing from a previous CNI. This catches @@ -146,6 +200,10 @@ if [ -d "$CNI_DIR" ]; then fi fi -log "Cilium ${CILIUM_VERSION} installed successfully." +if [ "$SKIP_INSTALL" -eq 1 ]; then + log "Cilium verification passed (install skipped, cluster was already ready)." +else + log "Cilium ${CILIUM_VERSION} installed successfully." +fi log "Cilium pod status:" kubectl get pods -n kube-system -l app.kubernetes.io/name=cilium-agent -o wide