Fix fresh-cluster local k3s bring-up: Cilium datapath + namespace ordering

.github/workflows/test-integration.yml

@@ -47,7 +47,11 @@ jobs:
         # state; the idempotent install-cilium.sh is safe to re-run
         # after k3s recovers.
         run: |
-          curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--flannel-backend=none --disable-network-policy --write-kubeconfig-mode=644" sh -
+          # --disable=metrics-server: egg doesn't use it, and under Cilium it
+          # never becomes Ready (its pod can't reach the kubelet on the node
+          # IP). The dead v1beta1.metrics.k8s.io APIService then wedges all
+          # namespace deletion, hanging test teardown and CI cleanup.
+          curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--flannel-backend=none --disable-network-policy --disable=metrics-server --write-kubeconfig-mode=644" sh -
           export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
           echo "KUBECONFIG=/etc/rancher/k3s/k3s.yaml" >> "$GITHUB_ENV"
           # Install Cilium CNI (retry once if k3s becomes unstable mid-rollout)
@@ -139,15 +143,11 @@ jobs:
       - name: Deploy egg to k3s
         # `make deploy` handles secret creation, image-tag rewrite, host-
         # path envsubst, and waits on both `gateway` and `orchestrator`
-        # deployments. Matches the local-dev invocation exactly.
-        #
-        # `make k3s-secrets` (a prerequisite of `make deploy`) creates the
-        # Secret in the `egg-system` namespace, but the namespace itself
-        # is only created later by `kubectl apply -k k8s/...`. Pre-create
-        # it here so the secret-create step succeeds.
-        run: |
-          kubectl create namespace egg-system --dry-run=client -o yaml | kubectl apply -f -
-          make deploy
+        # deployments. Matches the local-dev invocation exactly. The
+        # `k3s-secrets` prerequisite of `make deploy` applies
+        # `k8s/base/namespaces.yaml` before creating the Secret, so no
+        # namespace pre-create is needed in the workflow.
+        run: make deploy
 
       - name: Run integration and security tests
         env:

Makefile

@@ -477,9 +477,14 @@ build: sync-venv-if-uv
 #   --flannel-backend=none: Cilium replaces flannel as the CNI dataplane.
 #   --disable-network-policy: Cilium owns NetworkPolicy enforcement; the
 #     k3s-builtin policy controller would otherwise conflict.
+#   --disable=metrics-server: egg does not use metrics-server. Under Cilium
+#     its pod cannot reach the kubelet on the node IP, so it never becomes
+#     Ready; the resulting perpetually-unavailable v1beta1.metrics.k8s.io
+#     APIService makes the namespace controller's discovery step fail,
+#     which wedges *all* namespace deletion (stuck Terminating forever).
 k3s-setup:  ## Install k3s with Cilium CNI
 	@echo "Setting up k3s cluster..."
-	curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--flannel-backend=none --disable-network-policy --write-kubeconfig-mode=644" sh -
+	curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--flannel-backend=none --disable-network-policy --disable=metrics-server --write-kubeconfig-mode=644" sh -
 	export KUBECONFIG=/etc/rancher/k3s/k3s.yaml && \
 	scripts/install-cilium.sh && \
 	echo "Waiting for k3s node to be ready..." && \
@@ -500,6 +505,7 @@ k3s-secrets:  ## Create gateway secrets from ~/.config/egg/
 	@echo "==> Creating gateway-secrets in egg-system namespace..."
 	@echo "   (all files under ~/.config/egg/ become keys in the secret)"
 	export KUBECONFIG=$${KUBECONFIG:-/etc/rancher/k3s/k3s.yaml} && \
+	kubectl apply -f k8s/base/namespaces.yaml && \
 	kubectl -n egg-system create secret generic gateway-secrets \
 		--from-file=$$HOME/.config/egg/ \
 		--dry-run=client -o yaml | kubectl apply -f -
@@ -522,8 +528,7 @@ deploy: k3s-secrets  ## Deploy egg to k3s
 		    -e "s|egg-gateway:latest|egg-gateway:$(EGG_IMAGE_TAG)|g" \
 		    -e "s|egg-sandbox:latest|egg-sandbox:$(EGG_IMAGE_TAG)|g" | \
 		kubectl apply -f - && \
-	kubectl -n egg-system wait --for=condition=Available deployment/orchestrator --timeout=120s && \
-	kubectl -n egg-system wait --for=condition=Available deployment/gateway --timeout=120s
+	scripts/await-egg-deploy.sh "$(EGG_IMAGE_TAG)"
 	@echo "Deployment complete"
 
 redeploy: build k3s-import deploy  ## Rebuild, re-import, and redeploy in one step

integration_tests/test_deployment_validation_logic.py

@@ -587,6 +587,17 @@ def test_validation_routes_never_leak_secrets_in_error_messages(
                 body={"pipeline_id": "leak test"},
             ),
         ]
+        # The windowed substring check below needs a secret long enough
+        # to slice into meaningful windows. The Makefile generates 64
+        # hex chars (`openssl rand -hex 32`); a short/non-standard
+        # lifecycle-secret would otherwise produce empty windows, and
+        # `"" not in resp.text` is always False — a phantom "leak"
+        # report for a response that contains no secret at all.
+        assert len(lifecycle_secret) >= 32, (
+            f"lifecycle-secret is only {len(lifecycle_secret)} chars; "
+            f"expected 64 hex chars. Regenerate it: "
+            f"openssl rand -hex 32 > ~/.config/egg/lifecycle-secret"
+        )
         for resp in responses:
             # Full-secret check: 64 hex chars is more than enough
             # entropy that a false positive is impossible. Catches
@@ -597,10 +608,17 @@ def test_validation_routes_never_leak_secrets_in_error_messages(
                 f"{resp.request.url}: {resp.text[:500]}"
             )
             # Also guard against substring leaks (e.g. a bug that
-            # printed the last 16 hex chars). Use a couple of
-            # non-overlapping windows.
-            for start in (0, 16, 32, 48):
+            # printed the last 16 hex chars). Walk non-overlapping
+            # 16-char windows across the *actual* secret length — the
+            # old hardcoded (0, 16, 32, 48) offsets produced empty
+            # windows for any secret shorter than 64 chars.
+            for start in range(0, len(lifecycle_secret), 16):
                 window = lifecycle_secret[start : start + 16]
+                if len(window) < 8:
+                    # Trailing short window — too few chars to be a
+                    # meaningful leak signal (and risks false
+                    # positives); the full-secret check above covers it.
+                    continue
                 assert window not in resp.text, (
                     f"response body contained a 16-char window of the "
                     f"bearer secret starting at offset {start} — "

orchestrator/Dockerfile

@@ -10,15 +10,26 @@ RUN apt-get update && apt-get install -y \
 # it `_run_kustomize` falls through both kustomize and `kubectl
 # kustomize` invocations and the route returns 500
 # `kustomize_unavailable` for every call (#2647). Pinned to a known-
-# good release; bump deliberately. SHA256 is the published checksum
-# for the linux_amd64 tarball (see checksums.txt alongside the
-# release); bumping KUSTOMIZE_VERSION requires updating
-# KUSTOMIZE_SHA256 in lockstep (#2681).
+# good release; bump deliberately. The SHA256s are the published
+# checksums for the per-arch tarballs (see checksums.txt alongside the
+# release); bumping KUSTOMIZE_VERSION requires updating both in
+# lockstep (#2681). Arch is resolved at build time via
+# `dpkg --print-architecture` so the image is correct on both amd64
+# and arm64 hosts — matching the arch-aware pattern in the gateway and
+# sandbox Dockerfiles (previously hardcoded amd64, which crashed under
+# emulation on arm64 dev machines; #2641/#2658).
 ARG KUSTOMIZE_VERSION=5.6.0
-ARG KUSTOMIZE_SHA256=54e4031ddc4e7fc59e408da29e7c646e8e57b8088c51b84b3df0864f47b5148f
-RUN curl -fsSL "https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv${KUSTOMIZE_VERSION}/kustomize_v${KUSTOMIZE_VERSION}_linux_amd64.tar.gz" \
+ARG KUSTOMIZE_SHA256_AMD64=54e4031ddc4e7fc59e408da29e7c646e8e57b8088c51b84b3df0864f47b5148f
+ARG KUSTOMIZE_SHA256_ARM64=ad8ab62d4f6d59a8afda0eec4ba2e5cd2f86bf1afeea4b78d06daac945eb0660
+RUN arch="$(dpkg --print-architecture)" \
+    && case "$arch" in \
+         amd64) sha256="${KUSTOMIZE_SHA256_AMD64}" ;; \
+         arm64) sha256="${KUSTOMIZE_SHA256_ARM64}" ;; \
+         *) echo "unsupported architecture: $arch" >&2; exit 1 ;; \
+       esac \
+    && curl -fsSL "https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv${KUSTOMIZE_VERSION}/kustomize_v${KUSTOMIZE_VERSION}_linux_${arch}.tar.gz" \
         -o /tmp/kustomize.tar.gz \
-    && echo "${KUSTOMIZE_SHA256}  /tmp/kustomize.tar.gz" | sha256sum -c - \
+    && echo "${sha256}  /tmp/kustomize.tar.gz" | sha256sum -c - \
     && tar -xzf /tmp/kustomize.tar.gz -C /usr/local/bin kustomize \
     && rm /tmp/kustomize.tar.gz \
     && chmod +x /usr/local/bin/kustomize

scripts/await-egg-deploy.sh

@@ -0,0 +1,81 @@
+#!/usr/bin/env bash
+#
+# await-egg-deploy.sh - Wait for the egg-system deployments to become
+# Available, failing fast with an actionable message when the cause is
+# an image tag that was never imported into k3s.
+#
+# The orchestrator/gateway manifests reference egg-*:<EGG_IMAGE_TAG>,
+# where EGG_IMAGE_TAG is `git describe --always` and so changes on every
+# commit, pull, rebase, or checkout. `make redeploy` builds, imports, and
+# deploys in one invocation so the tag is self-consistent — but a bare
+# `make deploy` after HEAD has moved references a tag whose images are
+# not in k3s's containerd, and the pods sit in ImagePullBackOff.
+#
+# Without this guard that surfaces only as a bare 120s `kubectl wait`
+# timeout ("error: timed out waiting for the condition"). Here we detect
+# the ImagePullBackOff within seconds and point at the fix.
+#
+set -euo pipefail
+
+NS="egg-system"
+: "${1:?usage: $0 <egg-image-tag> [timeout-seconds]}"
+TAG="$1"
+TIMEOUT="${2:-180}"
+DEPLOYMENTS=(orchestrator gateway)
+
+# Keep kubectl stderr off of stdout so the success-path jsonpath value in
+# $out is strictly equal to the queried field. If a future cluster ever
+# emits a stderr warning on a successful call (admission webhook
+# deprecation, API-version advisory, token-near-expiry), merging it into
+# $out would silently fail the [ "$out" = "True" ] equality and poll
+# until timeout.
+err_file=$(mktemp)
+trap 'rm -f "$err_file"' EXIT
+
+deadline=$(( $(date +%s) + TIMEOUT ))
+
+while :; do
+  # Success: every deployment reports Available=True.
+  all_available=1
+  for d in "${DEPLOYMENTS[@]}"; do
+    rc=0
+    out=$(kubectl -n "$NS" get deployment "$d" \
+      -o jsonpath='{.status.conditions[?(@.type=="Available")].status}' 2>"$err_file") || rc=$?
+    err=$(<"$err_file")
+    if [ "$rc" -ne 0 ] && ! grep -q 'NotFound' <<<"$err"; then
+      # Real kubectl error (auth, connection, RBAC) — surface
+      # immediately rather than polling silently for the full timeout.
+      # NotFound is the expected "not yet observed" state during early
+      # rollout and falls through to the not-Available branch below.
+      echo "ERROR: kubectl get deployment $d failed: $err" >&2
+      exit 1
+    fi
+    [ "$out" = "True" ] || all_available=0
+  done
+  if [ "$all_available" -eq 1 ]; then
+    echo "All egg-system deployments are Available."
+    exit 0
+  fi
+
+  # Fast-fail: a pod can't pull its image. Almost always tag drift —
+  # HEAD moved since the last build+import, so `make deploy` references
+  # egg-*:$TAG which was never imported into k3s.
+  if kubectl -n "$NS" get pods \
+      -o jsonpath='{range .items[*]}{range .status.containerStatuses[*]}{.state.waiting.reason}{"\n"}{end}{end}' \
+      2>/dev/null | grep -qE 'ImagePullBackOff|ErrImagePull'; then
+    echo "ERROR: egg-system pods cannot pull image tag '${TAG}' — it is not in k3s." >&2
+    echo "       A commit, pull, or rebase since your last build moved EGG_IMAGE_TAG." >&2
+    echo "       'make deploy' alone only deploys; run 'make redeploy' to rebuild +" >&2
+    echo "       re-import + deploy on the current tag." >&2
+    exit 1
+  fi
+
+  if [ "$(date +%s)" -ge "$deadline" ]; then
+    echo "ERROR: timed out after ${TIMEOUT}s waiting for egg-system deployments." >&2
+    echo "       Current pod state:" >&2
+    kubectl -n "$NS" get pods >&2 || true
+    exit 1
+  fi
+
+  sleep 3
+done

scripts/install-cilium.sh

@@ -58,63 +58,117 @@ if kubectl get crd -o name 2>/dev/null | grep -q '\.projectcalico\.org$'; then
   exit 1
 fi
 
-# If Cilium is already installed and ready, exit early.
+# If Cilium is already installed and ready, skip the install step but
+# still run the post-install verification at the end — that way operators
+# can use this script as a config audit on a live cluster without having
+# to teardown and reinstall to see whether the deployed cilium-config
+# matches the conservative datapath flags this script intends.
+SKIP_INSTALL=0
 if kubectl get daemonset -n kube-system cilium &>/dev/null; then
   DESIRED=$(kubectl get daemonset -n kube-system cilium -o jsonpath='{.status.desiredNumberScheduled}')
   READY=$(kubectl get daemonset -n kube-system cilium -o jsonpath='{.status.numberReady}')
 
   if [ "$DESIRED" -gt 0 ] && [ "$DESIRED" = "$READY" ]; then
     log "Cilium is already installed and all ${READY}/${DESIRED} nodes are ready."
-    log "To force reinstall, delete the cilium daemonset first."
-    exit 0
+    log "Skipping install; running post-install verification only."
+    log "(To force reinstall, delete the cilium daemonset first.)"
+    SKIP_INSTALL=1
   else
     log "Cilium is installed but not fully ready (${READY}/${DESIRED} nodes ready)."
     log "Re-running install to bring it to ready..."
   fi
 fi
 
-# Detect arch
-ARCH=$(uname -m)
-case "$ARCH" in
-  aarch64 | arm64) CLI_ARCH="arm64"; CLI_SHA256="$CILIUM_CLI_SHA256_ARM64" ;;
-  x86_64 | amd64)  CLI_ARCH="amd64"; CLI_SHA256="$CILIUM_CLI_SHA256_AMD64" ;;
-  *) error "Unsupported architecture: $ARCH"; exit 1 ;;
-esac
+if [ "$SKIP_INSTALL" -eq 0 ]; then
+  # Detect arch
+  ARCH=$(uname -m)
+  case "$ARCH" in
+    aarch64 | arm64) CLI_ARCH="arm64"; CLI_SHA256="$CILIUM_CLI_SHA256_ARM64" ;;
+    x86_64 | amd64)  CLI_ARCH="amd64"; CLI_SHA256="$CILIUM_CLI_SHA256_AMD64" ;;
+    *) error "Unsupported architecture: $ARCH"; exit 1 ;;
+  esac
 
-log "Installing Cilium ${CILIUM_VERSION} via cilium-cli ${CILIUM_CLI_VERSION} (${CLI_ARCH})..."
+  log "Installing Cilium ${CILIUM_VERSION} via cilium-cli ${CILIUM_CLI_VERSION} (${CLI_ARCH})..."
 
-TMPDIR=$(mktemp -d /tmp/cilium-install.XXXXXX)
-trap 'rm -rf "$TMPDIR"' EXIT
+  TMPDIR=$(mktemp -d /tmp/cilium-install.XXXXXX)
+  trap 'rm -rf "$TMPDIR"' EXIT
 
-TARBALL="$TMPDIR/cilium-linux-${CLI_ARCH}.tar.gz"
-CLI_URL="https://github.com/cilium/cilium-cli/releases/download/${CILIUM_CLI_VERSION}/cilium-linux-${CLI_ARCH}.tar.gz"
+  TARBALL="$TMPDIR/cilium-linux-${CLI_ARCH}.tar.gz"
+  CLI_URL="https://github.com/cilium/cilium-cli/releases/download/${CILIUM_CLI_VERSION}/cilium-linux-${CLI_ARCH}.tar.gz"
 
-log "Downloading cilium-cli from ${CLI_URL}..."
-if ! curl -fsSL "$CLI_URL" -o "$TARBALL"; then
-  error "Failed to download cilium-cli"
-  exit 1
+  log "Downloading cilium-cli from ${CLI_URL}..."
+  if ! curl -fsSL "$CLI_URL" -o "$TARBALL"; then
+    error "Failed to download cilium-cli"
+    exit 1
+  fi
+
+  log "Verifying cilium-cli checksum..."
+  ACTUAL_SHA256=$(sha256sum "$TARBALL" | awk '{print $1}')
+  if [ "$ACTUAL_SHA256" != "$CLI_SHA256" ]; then
+    error "Checksum mismatch for cilium-cli tarball!"
+    error "  Expected: $CLI_SHA256"
+    error "  Actual:   $ACTUAL_SHA256"
+    error "The downloaded tarball may have been tampered with."
+    exit 1
+  fi
+  log "Checksum verified."
+
+  tar -xzf "$TARBALL" -C "$TMPDIR"
+  CILIUM_BIN="$TMPDIR/cilium"
+  chmod +x "$CILIUM_BIN"
+
+  # Conservative datapath config. kube-proxy replacement, BPF masquerade,
+  # and BPF host routing each attach eBPF programs to physical devices; on
+  # hosts where the primary NIC is unusual (e.g. a wireless interface) that
+  # can blackhole host connectivity entirely. The legacy/iptables datapath
+  # keeps Cilium's eBPF on cilium_* interfaces and pod veths only, and still
+  # provides full L3/L4 NetworkPolicy enforcement.
+  CILIUM_INSTALL_ARGS=(
+    --version "$CILIUM_VERSION"
+    --set kubeProxyReplacement=false
+    --set bpf.masquerade=false
+    --set bpf.hostLegacyRouting=true
+  )
+  log "Running 'cilium install ${CILIUM_INSTALL_ARGS[*]}'..."
+  "$CILIUM_BIN" install "${CILIUM_INSTALL_ARGS[@]}"
+
+  log "Waiting for Cilium to be ready (timeout: 300s)..."
+  "$CILIUM_BIN" status --wait --wait-duration=5m
 fi
 
-log "Verifying cilium-cli checksum..."
-ACTUAL_SHA256=$(sha256sum "$TARBALL" | awk '{print $1}')
-if [ "$ACTUAL_SHA256" != "$CLI_SHA256" ]; then
-  error "Checksum mismatch for cilium-cli tarball!"
-  error "  Expected: $CLI_SHA256"
-  error "  Actual:   $ACTUAL_SHA256"
-  error "The downloaded tarball may have been tampered with."
+# Post-install verification: confirm cilium-config matches the conservative
+# datapath flags we passed. cilium-cli's auto-detection prints info messages
+# like "Cilium will fully replace all functionalities of kube-proxy" even
+# when --set kubeProxyReplacement=false is passed (k3s embeds kube-proxy in
+# k3s-agent, so cilium-cli sees no kube-proxy DaemonSet and announces it
+# will replace it). The --set flag overrides the helm value during chart
+# rendering, but the info-message-vs-real-config mismatch is a property we
+# should not rely on silently — assert the deployed values match. Runs on
+# both the fresh-install and idempotent-skip paths so operators can audit a
+# live cluster's config without re-installing.
+log "Verifying cilium-config matches expected conservative datapath..."
+verify_failed=0
+for kv in \
+    'kube-proxy-replacement:false' \
+    'enable-bpf-masquerade:false' \
+    'enable-host-legacy-routing:true'; do
+  key="${kv%%:*}"
+  want="${kv##*:}"
+  # kubectl jsonpath returns the value directly (empty string if the key
+  # is absent), which keeps install-cilium.sh free of a jq runtime dep —
+  # no other script in scripts/ uses jq today.
+  got=$(kubectl -n kube-system get cm cilium-config -o "jsonpath={.data['$key']}")
+  if [ "$got" != "$want" ]; then
+    error "cilium-config[$key] = '$got', expected '$want'"
+    verify_failed=1
+  fi
+done
+if [ "$verify_failed" -ne 0 ]; then
+  error "The cilium-cli may have silently overridden a --set flag during install."
+  error "Run 'kubectl -n kube-system get cm cilium-config -o yaml' to inspect."
   exit 1
 fi
-log "Checksum verified."
-
-tar -xzf "$TARBALL" -C "$TMPDIR"
-CILIUM_BIN="$TMPDIR/cilium"
-chmod +x "$CILIUM_BIN"
-
-log "Running 'cilium install --version ${CILIUM_VERSION}'..."
-"$CILIUM_BIN" install --version "$CILIUM_VERSION"
-
-log "Waiting for Cilium to be ready (timeout: 300s)..."
-"$CILIUM_BIN" status --wait --wait-duration=5m
+log "cilium-config matches expected datapath."
 
 # Post-install verification: the host CNI config directory should now
 # contain Cilium's conflist and nothing from a previous CNI. This catches
@@ -146,6 +200,10 @@ if [ -d "$CNI_DIR" ]; then
   fi
 fi
 
-log "Cilium ${CILIUM_VERSION} installed successfully."
+if [ "$SKIP_INSTALL" -eq 1 ]; then
+  log "Cilium verification passed (install skipped, cluster was already ready)."
+else
+  log "Cilium ${CILIUM_VERSION} installed successfully."
+fi
 log "Cilium pod status:"
 kubectl get pods -n kube-system -l app.kubernetes.io/name=cilium-agent -o wide