From e3077289bb08a9105d7ebe55988b68170ee23ea9 Mon Sep 17 00:00:00 2001
From: James Wiesebron <jameswiesebron@khanacademy.org>
Date: Thu, 14 May 2026 13:15:57 -0700
Subject: [PATCH 1/8] Fix fresh-cluster local k3s bring-up: Cilium datapath +
 namespace ordering

Two independent bugs both block bringing up a local k3s cluster from
scratch (make k3s-teardown && make k3s-setup && make deploy):

install-cilium.sh: the bare `cilium install` from #2704 enables
kube-proxy replacement, BPF masquerade, and BPF host routing, all of
which attach eBPF programs to the host's primary NIC. On hosts where
that NIC is a wireless interface, this blackholes host connectivity
entirely. Pass conservative datapath flags (kubeProxyReplacement=false,
bpf.masquerade=false, bpf.hostLegacyRouting=true) so Cilium's eBPF stays
on cilium_* interfaces and pod veths; full L3/L4 NetworkPolicy
enforcement is unaffected.

Makefile: k3s-secrets created the gateway-secrets secret in the
egg-system namespace, but that namespace is created by deploy's manifest
apply, which runs after k3s-secrets (deploy: k3s-secrets). On an existing
cluster the namespace is already present; on a fresh cluster k3s-secrets
fails with `namespaces "egg-system" not found`. Apply
k8s/base/namespaces.yaml in k3s-secrets before creating the secret.
---
 Makefile                  |  1 +
 scripts/install-cilium.sh | 16 ++++++++++++++--
 2 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index cda214f468..afca9ae37f 100644
--- a/Makefile
+++ b/Makefile
@@ -500,6 +500,7 @@ k3s-secrets:  ## Create gateway secrets from ~/.config/egg/
 	@echo "==> Creating gateway-secrets in egg-system namespace..."
 	@echo "   (all files under ~/.config/egg/ become keys in the secret)"
 	export KUBECONFIG=$${KUBECONFIG:-/etc/rancher/k3s/k3s.yaml} && \
+	kubectl apply -f k8s/base/namespaces.yaml && \
 	kubectl -n egg-system create secret generic gateway-secrets \
 		--from-file=$$HOME/.config/egg/ \
 		--dry-run=client -o yaml | kubectl apply -f -
diff --git a/scripts/install-cilium.sh b/scripts/install-cilium.sh
index 82605ad76f..bb3f3e4c7e 100755
--- a/scripts/install-cilium.sh
+++ b/scripts/install-cilium.sh
@@ -110,8 +110,20 @@ tar -xzf "$TARBALL" -C "$TMPDIR"
 CILIUM_BIN="$TMPDIR/cilium"
 chmod +x "$CILIUM_BIN"
 
-log "Running 'cilium install --version ${CILIUM_VERSION}'..."
-"$CILIUM_BIN" install --version "$CILIUM_VERSION"
+# Conservative datapath config. kube-proxy replacement, BPF masquerade,
+# and BPF host routing each attach eBPF programs to physical devices; on
+# hosts where the primary NIC is unusual (e.g. a wireless interface) that
+# can blackhole host connectivity entirely. The legacy/iptables datapath
+# keeps Cilium's eBPF on cilium_* interfaces and pod veths only, and still
+# provides full L3/L4 NetworkPolicy enforcement.
+CILIUM_INSTALL_ARGS=(
+  --version "$CILIUM_VERSION"
+  --set kubeProxyReplacement=false
+  --set bpf.masquerade=false
+  --set bpf.hostLegacyRouting=true
+)
+log "Running 'cilium install ${CILIUM_INSTALL_ARGS[*]}'..."
+"$CILIUM_BIN" install "${CILIUM_INSTALL_ARGS[@]}"
 
 log "Waiting for Cilium to be ready (timeout: 300s)..."
 "$CILIUM_BIN" status --wait --wait-duration=5m

From cd8d83e0fd6b11c8ed7da810e31c74fb356ded25 Mon Sep 17 00:00:00 2001
From: "egg-reviewer[bot]"
 <261018737+egg-reviewer[bot]@users.noreply.github.com>
Date: Thu, 14 May 2026 21:26:32 +0000
Subject: [PATCH 2/8] Verify cilium-config matches expected datapath after
 install

Post-install check reads the cilium-config ConfigMap and asserts that
kube-proxy-replacement, enable-bpf-masquerade, and enable-host-legacy-routing
match the values we passed via --set. The cilium-cli's auto-detection
prints 'Cilium will fully replace all functionalities of kube-proxy' even
when --set kubeProxyReplacement=false is passed (k3s embeds kube-proxy in
k3s-agent, so cilium-cli sees no kube-proxy DaemonSet and announces it
will replace it). The --set flag overrides during chart rendering, but
the info-message-vs-real-config mismatch is a property we should not
rely on silently. Fails fast if a future cilium-cli release ever changes
override precedence.
---
 scripts/install-cilium.sh | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/scripts/install-cilium.sh b/scripts/install-cilium.sh
index bb3f3e4c7e..6fd7bb25f8 100755
--- a/scripts/install-cilium.sh
+++ b/scripts/install-cilium.sh
@@ -128,6 +128,36 @@ log "Running 'cilium install ${CILIUM_INSTALL_ARGS[*]}'..."
 log "Waiting for Cilium to be ready (timeout: 300s)..."
 "$CILIUM_BIN" status --wait --wait-duration=5m
 
+# Post-install verification: confirm cilium-config matches the conservative
+# datapath flags we passed. cilium-cli's auto-detection prints info messages
+# like "Cilium will fully replace all functionalities of kube-proxy" even
+# when --set kubeProxyReplacement=false is passed (k3s embeds kube-proxy in
+# k3s-agent, so cilium-cli sees no kube-proxy DaemonSet and announces it
+# will replace it). The --set flag overrides the helm value during chart
+# rendering, but the info-message-vs-real-config mismatch is a property we
+# should not rely on silently — assert the deployed values match.
+log "Verifying cilium-config matches expected conservative datapath..."
+CFG=$(kubectl -n kube-system get cm cilium-config -o json)
+verify_failed=0
+for kv in \
+    'kube-proxy-replacement:false' \
+    'enable-bpf-masquerade:false' \
+    'enable-host-legacy-routing:true'; do
+  key="${kv%%:*}"
+  want="${kv##*:}"
+  got=$(echo "$CFG" | jq -r ".data[\"$key\"] // empty")
+  if [ "$got" != "$want" ]; then
+    error "cilium-config[$key] = '$got', expected '$want'"
+    verify_failed=1
+  fi
+done
+if [ "$verify_failed" -ne 0 ]; then
+  error "The cilium-cli may have silently overridden a --set flag during install."
+  error "Run 'kubectl -n kube-system get cm cilium-config -o yaml' to inspect."
+  exit 1
+fi
+log "cilium-config matches expected datapath."
+
 # Post-install verification: the host CNI config directory should now
 # contain Cilium's conflist and nothing from a previous CNI. This catches
 # the silent-failure mode where kubelet keeps using a leftover Calico

From 1adf6d6085ad4412f2d243bf03910b454c560f92 Mon Sep 17 00:00:00 2001
From: James Wiesebron <jameswiesebron@khanacademy.org>
Date: Thu, 14 May 2026 14:27:49 -0700
Subject: [PATCH 3/8] Disable metrics-server in k3s install to fix namespace-GC
 wedge

Under the Cilium CNI (#2704), the k3s-bundled metrics-server pod cannot
reach the kubelet on the node IP, so its readiness probe never passes
and it stays out of its Service's endpoints. The v1beta1.metrics.k8s.io
APIService is therefore permanently unavailable.

The namespace controller runs API discovery across all groups before
finalizing any namespace; a down APIService makes discovery fail
(NamespaceDeletionDiscoveryFailure), so *every* namespace deletion hangs
in Terminating forever. In CI this wedged integration-test fixture
teardown (kubectl delete namespace timed out) and the job-cleanup step,
blowing past the 30-minute job limit.

egg does not use metrics-server (no HPA, no kubectl top, no
metrics.k8s.io consumers), so disable it in both the Makefile k3s-setup
target and the test-integration workflow's k3s install. This removes the
unused addon and the cluster-wide failure mode it introduced.
---
 .github/workflows/test-integration.yml | 6 +++++-
 Makefile                               | 7 ++++++-
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test-integration.yml b/.github/workflows/test-integration.yml
index ca0e3aef46..b7fe9f7eeb 100644
--- a/.github/workflows/test-integration.yml
+++ b/.github/workflows/test-integration.yml
@@ -47,7 +47,11 @@ jobs:
         # state; the idempotent install-cilium.sh is safe to re-run
         # after k3s recovers.
         run: |
-          curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--flannel-backend=none --disable-network-policy --write-kubeconfig-mode=644" sh -
+          # --disable=metrics-server: egg doesn't use it, and under Cilium it
+          # never becomes Ready (its pod can't reach the kubelet on the node
+          # IP). The dead v1beta1.metrics.k8s.io APIService then wedges all
+          # namespace deletion, hanging test teardown and CI cleanup.
+          curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--flannel-backend=none --disable-network-policy --disable=metrics-server --write-kubeconfig-mode=644" sh -
           export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
           echo "KUBECONFIG=/etc/rancher/k3s/k3s.yaml" >> "$GITHUB_ENV"
           # Install Cilium CNI (retry once if k3s becomes unstable mid-rollout)
diff --git a/Makefile b/Makefile
index afca9ae37f..2aea7de372 100644
--- a/Makefile
+++ b/Makefile
@@ -477,9 +477,14 @@ build: sync-venv-if-uv
 #   --flannel-backend=none: Cilium replaces flannel as the CNI dataplane.
 #   --disable-network-policy: Cilium owns NetworkPolicy enforcement; the
 #     k3s-builtin policy controller would otherwise conflict.
+#   --disable=metrics-server: egg does not use metrics-server. Under Cilium
+#     its pod cannot reach the kubelet on the node IP, so it never becomes
+#     Ready; the resulting perpetually-unavailable v1beta1.metrics.k8s.io
+#     APIService makes the namespace controller's discovery step fail,
+#     which wedges *all* namespace deletion (stuck Terminating forever).
 k3s-setup:  ## Install k3s with Cilium CNI
 	@echo "Setting up k3s cluster..."
-	curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--flannel-backend=none --disable-network-policy --write-kubeconfig-mode=644" sh -
+	curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--flannel-backend=none --disable-network-policy --disable=metrics-server --write-kubeconfig-mode=644" sh -
 	export KUBECONFIG=/etc/rancher/k3s/k3s.yaml && \
 	scripts/install-cilium.sh && \
 	echo "Waiting for k3s node to be ready..." && \

From f1a9ee27463d45ce507f9294391ba54d63468ec9 Mon Sep 17 00:00:00 2001
From: James Wiesebron <jameswiesebron@khanacademy.org>
Date: Thu, 14 May 2026 14:34:44 -0700
Subject: [PATCH 4/8] Fix arm64 kustomize crash and harden secret-leak test
 window check
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

orchestrator/Dockerfile hardcoded the linux_amd64 kustomize tarball. On
an arm64 host `make build` produces an arm64 orchestrator image (the
python:3.14-slim base is multi-arch) with an amd64 kustomize binary
inside it; it runs under emulation and the amd64 Go runtime crashes. CI
never caught it — CI runners are amd64. Resolve the arch at build time
via `dpkg --print-architecture` (matching the gateway and sandbox
Dockerfiles) and pin per-arch SHA256s.

test_deployment_validation_logic.py's secret-leak check sliced the
bearer at hardcoded offsets 0/16/32/48, assuming a 64-hex-char secret.
A shorter lifecycle-secret yields empty windows, and `"" not in text`
is always False — reporting a phantom leak for a response that contains
no secret at all. Walk windows across the actual secret length, skip
trailing short windows, and assert a minimum length up front with a
clear "regenerate it" message.
---
 .../test_deployment_validation_logic.py       | 24 +++++++++++++++---
 orchestrator/Dockerfile                       | 25 +++++++++++++------
 2 files changed, 39 insertions(+), 10 deletions(-)

diff --git a/integration_tests/test_deployment_validation_logic.py b/integration_tests/test_deployment_validation_logic.py
index 9ad1c96a46..8ab2e56b81 100644
--- a/integration_tests/test_deployment_validation_logic.py
+++ b/integration_tests/test_deployment_validation_logic.py
@@ -587,6 +587,17 @@ def test_validation_routes_never_leak_secrets_in_error_messages(
                 body={"pipeline_id": "leak test"},
             ),
         ]
+        # The windowed substring check below needs a secret long enough
+        # to slice into meaningful windows. The Makefile generates 64
+        # hex chars (`openssl rand -hex 32`); a short/non-standard
+        # lifecycle-secret would otherwise produce empty windows, and
+        # `"" not in resp.text` is always False — a phantom "leak"
+        # report for a response that contains no secret at all.
+        assert len(lifecycle_secret) >= 32, (
+            f"lifecycle-secret is only {len(lifecycle_secret)} chars; "
+            f"expected 64 hex chars. Regenerate it: "
+            f"openssl rand -hex 32 > ~/.config/egg/lifecycle-secret"
+        )
         for resp in responses:
             # Full-secret check: 64 hex chars is more than enough
             # entropy that a false positive is impossible. Catches
@@ -597,10 +608,17 @@ def test_validation_routes_never_leak_secrets_in_error_messages(
                 f"{resp.request.url}: {resp.text[:500]}"
             )
             # Also guard against substring leaks (e.g. a bug that
-            # printed the last 16 hex chars). Use a couple of
-            # non-overlapping windows.
-            for start in (0, 16, 32, 48):
+            # printed the last 16 hex chars). Walk non-overlapping
+            # 16-char windows across the *actual* secret length — the
+            # old hardcoded (0, 16, 32, 48) offsets produced empty
+            # windows for any secret shorter than 64 chars.
+            for start in range(0, len(lifecycle_secret), 16):
                 window = lifecycle_secret[start : start + 16]
+                if len(window) < 8:
+                    # Trailing short window — too few chars to be a
+                    # meaningful leak signal (and risks false
+                    # positives); the full-secret check above covers it.
+                    continue
                 assert window not in resp.text, (
                     f"response body contained a 16-char window of the "
                     f"bearer secret starting at offset {start} — "
diff --git a/orchestrator/Dockerfile b/orchestrator/Dockerfile
index 56aeb8e90c..aec881048e 100644
--- a/orchestrator/Dockerfile
+++ b/orchestrator/Dockerfile
@@ -10,15 +10,26 @@ RUN apt-get update && apt-get install -y \
 # it `_run_kustomize` falls through both kustomize and `kubectl
 # kustomize` invocations and the route returns 500
 # `kustomize_unavailable` for every call (#2647). Pinned to a known-
-# good release; bump deliberately. SHA256 is the published checksum
-# for the linux_amd64 tarball (see checksums.txt alongside the
-# release); bumping KUSTOMIZE_VERSION requires updating
-# KUSTOMIZE_SHA256 in lockstep (#2681).
+# good release; bump deliberately. The SHA256s are the published
+# checksums for the per-arch tarballs (see checksums.txt alongside the
+# release); bumping KUSTOMIZE_VERSION requires updating both in
+# lockstep (#2681). Arch is resolved at build time via
+# `dpkg --print-architecture` so the image is correct on both amd64
+# and arm64 hosts — matching the arch-aware pattern in the gateway and
+# sandbox Dockerfiles (previously hardcoded amd64, which crashed under
+# emulation on arm64 dev machines; #2641/#2658).
 ARG KUSTOMIZE_VERSION=5.6.0
-ARG KUSTOMIZE_SHA256=54e4031ddc4e7fc59e408da29e7c646e8e57b8088c51b84b3df0864f47b5148f
-RUN curl -fsSL "https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv${KUSTOMIZE_VERSION}/kustomize_v${KUSTOMIZE_VERSION}_linux_amd64.tar.gz" \
+ARG KUSTOMIZE_SHA256_AMD64=54e4031ddc4e7fc59e408da29e7c646e8e57b8088c51b84b3df0864f47b5148f
+ARG KUSTOMIZE_SHA256_ARM64=ad8ab62d4f6d59a8afda0eec4ba2e5cd2f86bf1afeea4b78d06daac945eb0660
+RUN arch="$(dpkg --print-architecture)" \
+    && case "$arch" in \
+         amd64) sha256="${KUSTOMIZE_SHA256_AMD64}" ;; \
+         arm64) sha256="${KUSTOMIZE_SHA256_ARM64}" ;; \
+         *) echo "unsupported architecture: $arch" >&2; exit 1 ;; \
+       esac \
+    && curl -fsSL "https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv${KUSTOMIZE_VERSION}/kustomize_v${KUSTOMIZE_VERSION}_linux_${arch}.tar.gz" \
         -o /tmp/kustomize.tar.gz \
-    && echo "${KUSTOMIZE_SHA256}  /tmp/kustomize.tar.gz" | sha256sum -c - \
+    && echo "${sha256}  /tmp/kustomize.tar.gz" | sha256sum -c - \
     && tar -xzf /tmp/kustomize.tar.gz -C /usr/local/bin kustomize \
     && rm /tmp/kustomize.tar.gz \
     && chmod +x /usr/local/bin/kustomize

From 7b139030f882bbedfb6868ca0359989d0c9800b3 Mon Sep 17 00:00:00 2001
From: "egg-reviewer[bot]"
 <261018737+egg-reviewer[bot]@users.noreply.github.com>
Date: Thu, 14 May 2026 21:48:01 +0000
Subject: [PATCH 5/8] Address re-review feedback: drop jq, audit-mode verify,
 drop CI pre-create

- install-cilium.sh: replace jq with kubectl jsonpath for cilium-config
  read-back. No other script in scripts/ depends on jq, and jq is not
  checked as a prereq in this script, so a missing jq would fail after
  'cilium install' has already committed real cluster state. jsonpath
  returns the value directly (empty for absent keys), preserving the
  three-flag assertion verbatim.
- install-cilium.sh: run the cilium-config verification block on both
  the fresh-install and idempotent-skip paths. Operators can now use
  this script as a live-cluster config audit without teardown +
  reinstall. The CNI on-host check also runs on the skip path; the
  final log line distinguishes "installed" vs "verified (skipped)".
- test-integration.yml: drop the redundant 'kubectl create namespace
  egg-system' pre-create. The Makefile k3s-secrets target now applies
  k8s/base/namespaces.yaml before creating the Secret, so the workflow
  no longer needs the workaround. Comment updated to match.
---
 .github/workflows/test-integration.yml |  14 +--
 scripts/install-cilium.sh              | 126 ++++++++++++++-----------
 2 files changed, 76 insertions(+), 64 deletions(-)

diff --git a/.github/workflows/test-integration.yml b/.github/workflows/test-integration.yml
index b7fe9f7eeb..a22d7fe3c8 100644
--- a/.github/workflows/test-integration.yml
+++ b/.github/workflows/test-integration.yml
@@ -143,15 +143,11 @@ jobs:
       - name: Deploy egg to k3s
         # `make deploy` handles secret creation, image-tag rewrite, host-
         # path envsubst, and waits on both `gateway` and `orchestrator`
-        # deployments. Matches the local-dev invocation exactly.
-        #
-        # `make k3s-secrets` (a prerequisite of `make deploy`) creates the
-        # Secret in the `egg-system` namespace, but the namespace itself
-        # is only created later by `kubectl apply -k k8s/...`. Pre-create
-        # it here so the secret-create step succeeds.
-        run: |
-          kubectl create namespace egg-system --dry-run=client -o yaml | kubectl apply -f -
-          make deploy
+        # deployments. Matches the local-dev invocation exactly. The
+        # `k3s-secrets` prerequisite of `make deploy` applies
+        # `k8s/base/namespaces.yaml` before creating the Secret, so no
+        # namespace pre-create is needed in the workflow.
+        run: make deploy
 
       - name: Run integration and security tests
         env:
diff --git a/scripts/install-cilium.sh b/scripts/install-cilium.sh
index 6fd7bb25f8..de8b287a40 100755
--- a/scripts/install-cilium.sh
+++ b/scripts/install-cilium.sh
@@ -58,75 +58,83 @@ if kubectl get crd -o name 2>/dev/null | grep -q '\.projectcalico\.org$'; then
   exit 1
 fi
 
-# If Cilium is already installed and ready, exit early.
+# If Cilium is already installed and ready, skip the install step but
+# still run the post-install verification at the end — that way operators
+# can use this script as a config audit on a live cluster without having
+# to teardown and reinstall to see whether the deployed cilium-config
+# matches the conservative datapath flags this script intends.
+SKIP_INSTALL=0
 if kubectl get daemonset -n kube-system cilium &>/dev/null; then
   DESIRED=$(kubectl get daemonset -n kube-system cilium -o jsonpath='{.status.desiredNumberScheduled}')
   READY=$(kubectl get daemonset -n kube-system cilium -o jsonpath='{.status.numberReady}')
 
   if [ "$DESIRED" -gt 0 ] && [ "$DESIRED" = "$READY" ]; then
     log "Cilium is already installed and all ${READY}/${DESIRED} nodes are ready."
-    log "To force reinstall, delete the cilium daemonset first."
-    exit 0
+    log "Skipping install; running post-install verification only."
+    log "(To force reinstall, delete the cilium daemonset first.)"
+    SKIP_INSTALL=1
   else
     log "Cilium is installed but not fully ready (${READY}/${DESIRED} nodes ready)."
     log "Re-running install to bring it to ready..."
   fi
 fi
 
-# Detect arch
-ARCH=$(uname -m)
-case "$ARCH" in
-  aarch64 | arm64) CLI_ARCH="arm64"; CLI_SHA256="$CILIUM_CLI_SHA256_ARM64" ;;
-  x86_64 | amd64)  CLI_ARCH="amd64"; CLI_SHA256="$CILIUM_CLI_SHA256_AMD64" ;;
-  *) error "Unsupported architecture: $ARCH"; exit 1 ;;
-esac
+if [ "$SKIP_INSTALL" -eq 0 ]; then
+  # Detect arch
+  ARCH=$(uname -m)
+  case "$ARCH" in
+    aarch64 | arm64) CLI_ARCH="arm64"; CLI_SHA256="$CILIUM_CLI_SHA256_ARM64" ;;
+    x86_64 | amd64)  CLI_ARCH="amd64"; CLI_SHA256="$CILIUM_CLI_SHA256_AMD64" ;;
+    *) error "Unsupported architecture: $ARCH"; exit 1 ;;
+  esac
 
-log "Installing Cilium ${CILIUM_VERSION} via cilium-cli ${CILIUM_CLI_VERSION} (${CLI_ARCH})..."
+  log "Installing Cilium ${CILIUM_VERSION} via cilium-cli ${CILIUM_CLI_VERSION} (${CLI_ARCH})..."
 
-TMPDIR=$(mktemp -d /tmp/cilium-install.XXXXXX)
-trap 'rm -rf "$TMPDIR"' EXIT
+  TMPDIR=$(mktemp -d /tmp/cilium-install.XXXXXX)
+  trap 'rm -rf "$TMPDIR"' EXIT
 
-TARBALL="$TMPDIR/cilium-linux-${CLI_ARCH}.tar.gz"
-CLI_URL="https://github.com/cilium/cilium-cli/releases/download/${CILIUM_CLI_VERSION}/cilium-linux-${CLI_ARCH}.tar.gz"
+  TARBALL="$TMPDIR/cilium-linux-${CLI_ARCH}.tar.gz"
+  CLI_URL="https://github.com/cilium/cilium-cli/releases/download/${CILIUM_CLI_VERSION}/cilium-linux-${CLI_ARCH}.tar.gz"
 
-log "Downloading cilium-cli from ${CLI_URL}..."
-if ! curl -fsSL "$CLI_URL" -o "$TARBALL"; then
-  error "Failed to download cilium-cli"
-  exit 1
-fi
+  log "Downloading cilium-cli from ${CLI_URL}..."
+  if ! curl -fsSL "$CLI_URL" -o "$TARBALL"; then
+    error "Failed to download cilium-cli"
+    exit 1
+  fi
 
-log "Verifying cilium-cli checksum..."
-ACTUAL_SHA256=$(sha256sum "$TARBALL" | awk '{print $1}')
-if [ "$ACTUAL_SHA256" != "$CLI_SHA256" ]; then
-  error "Checksum mismatch for cilium-cli tarball!"
-  error "  Expected: $CLI_SHA256"
-  error "  Actual:   $ACTUAL_SHA256"
-  error "The downloaded tarball may have been tampered with."
-  exit 1
+  log "Verifying cilium-cli checksum..."
+  ACTUAL_SHA256=$(sha256sum "$TARBALL" | awk '{print $1}')
+  if [ "$ACTUAL_SHA256" != "$CLI_SHA256" ]; then
+    error "Checksum mismatch for cilium-cli tarball!"
+    error "  Expected: $CLI_SHA256"
+    error "  Actual:   $ACTUAL_SHA256"
+    error "The downloaded tarball may have been tampered with."
+    exit 1
+  fi
+  log "Checksum verified."
+
+  tar -xzf "$TARBALL" -C "$TMPDIR"
+  CILIUM_BIN="$TMPDIR/cilium"
+  chmod +x "$CILIUM_BIN"
+
+  # Conservative datapath config. kube-proxy replacement, BPF masquerade,
+  # and BPF host routing each attach eBPF programs to physical devices; on
+  # hosts where the primary NIC is unusual (e.g. a wireless interface) that
+  # can blackhole host connectivity entirely. The legacy/iptables datapath
+  # keeps Cilium's eBPF on cilium_* interfaces and pod veths only, and still
+  # provides full L3/L4 NetworkPolicy enforcement.
+  CILIUM_INSTALL_ARGS=(
+    --version "$CILIUM_VERSION"
+    --set kubeProxyReplacement=false
+    --set bpf.masquerade=false
+    --set bpf.hostLegacyRouting=true
+  )
+  log "Running 'cilium install ${CILIUM_INSTALL_ARGS[*]}'..."
+  "$CILIUM_BIN" install "${CILIUM_INSTALL_ARGS[@]}"
+
+  log "Waiting for Cilium to be ready (timeout: 300s)..."
+  "$CILIUM_BIN" status --wait --wait-duration=5m
 fi
-log "Checksum verified."
-
-tar -xzf "$TARBALL" -C "$TMPDIR"
-CILIUM_BIN="$TMPDIR/cilium"
-chmod +x "$CILIUM_BIN"
-
-# Conservative datapath config. kube-proxy replacement, BPF masquerade,
-# and BPF host routing each attach eBPF programs to physical devices; on
-# hosts where the primary NIC is unusual (e.g. a wireless interface) that
-# can blackhole host connectivity entirely. The legacy/iptables datapath
-# keeps Cilium's eBPF on cilium_* interfaces and pod veths only, and still
-# provides full L3/L4 NetworkPolicy enforcement.
-CILIUM_INSTALL_ARGS=(
-  --version "$CILIUM_VERSION"
-  --set kubeProxyReplacement=false
-  --set bpf.masquerade=false
-  --set bpf.hostLegacyRouting=true
-)
-log "Running 'cilium install ${CILIUM_INSTALL_ARGS[*]}'..."
-"$CILIUM_BIN" install "${CILIUM_INSTALL_ARGS[@]}"
-
-log "Waiting for Cilium to be ready (timeout: 300s)..."
-"$CILIUM_BIN" status --wait --wait-duration=5m
 
 # Post-install verification: confirm cilium-config matches the conservative
 # datapath flags we passed. cilium-cli's auto-detection prints info messages
@@ -135,9 +143,10 @@ log "Waiting for Cilium to be ready (timeout: 300s)..."
 # k3s-agent, so cilium-cli sees no kube-proxy DaemonSet and announces it
 # will replace it). The --set flag overrides the helm value during chart
 # rendering, but the info-message-vs-real-config mismatch is a property we
-# should not rely on silently — assert the deployed values match.
+# should not rely on silently — assert the deployed values match. Runs on
+# both the fresh-install and idempotent-skip paths so operators can audit a
+# live cluster's config without re-installing.
 log "Verifying cilium-config matches expected conservative datapath..."
-CFG=$(kubectl -n kube-system get cm cilium-config -o json)
 verify_failed=0
 for kv in \
     'kube-proxy-replacement:false' \
@@ -145,7 +154,10 @@ for kv in \
     'enable-host-legacy-routing:true'; do
   key="${kv%%:*}"
   want="${kv##*:}"
-  got=$(echo "$CFG" | jq -r ".data[\"$key\"] // empty")
+  # kubectl jsonpath returns the value directly (empty string if the key
+  # is absent), which keeps install-cilium.sh free of a jq runtime dep —
+  # no other script in scripts/ uses jq today.
+  got=$(kubectl -n kube-system get cm cilium-config -o "jsonpath={.data['$key']}")
   if [ "$got" != "$want" ]; then
     error "cilium-config[$key] = '$got', expected '$want'"
     verify_failed=1
@@ -188,6 +200,10 @@ if [ -d "$CNI_DIR" ]; then
   fi
 fi
 
-log "Cilium ${CILIUM_VERSION} installed successfully."
+if [ "$SKIP_INSTALL" -eq 1 ]; then
+  log "Cilium verification passed (install skipped, cluster was already ready)."
+else
+  log "Cilium ${CILIUM_VERSION} installed successfully."
+fi
 log "Cilium pod status:"
 kubectl get pods -n kube-system -l app.kubernetes.io/name=cilium-agent -o wide

From e3ceeb391279cfde23ee3422089ebe53b10a5b39 Mon Sep 17 00:00:00 2001
From: James Wiesebron <jameswiesebron@khanacademy.org>
Date: Thu, 14 May 2026 17:05:34 -0700
Subject: [PATCH 6/8] deploy: fail fast on un-imported image tags instead of a
 120s wait
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

`make deploy` references egg-*:$(EGG_IMAGE_TAG), where EGG_IMAGE_TAG is
`git describe --always` and so changes on every commit, pull, rebase, or
checkout. `make redeploy` builds + imports + deploys in one invocation
so the tag stays self-consistent, but a bare `make deploy` after HEAD
has moved references images that were never imported into k3s — the
pods then sit in ImagePullBackOff.

Previously that surfaced only as a bare 120s `kubectl wait` timeout
("error: timed out waiting for the condition") with no hint at the
cause. Replace the two per-deployment waits with await-egg-deploy.sh,
which polls for Available and fails within seconds when it sees an
ImagePullBackOff, naming the tag and pointing at `make redeploy`.
---
 Makefile                    |  3 +-
 scripts/await-egg-deploy.sh | 61 +++++++++++++++++++++++++++++++++++++
 2 files changed, 62 insertions(+), 2 deletions(-)
 create mode 100755 scripts/await-egg-deploy.sh

diff --git a/Makefile b/Makefile
index 2aea7de372..eaf4e106ae 100644
--- a/Makefile
+++ b/Makefile
@@ -528,8 +528,7 @@ deploy: k3s-secrets  ## Deploy egg to k3s
 		    -e "s|egg-gateway:latest|egg-gateway:$(EGG_IMAGE_TAG)|g" \
 		    -e "s|egg-sandbox:latest|egg-sandbox:$(EGG_IMAGE_TAG)|g" | \
 		kubectl apply -f - && \
-	kubectl -n egg-system wait --for=condition=Available deployment/orchestrator --timeout=120s && \
-	kubectl -n egg-system wait --for=condition=Available deployment/gateway --timeout=120s
+	scripts/await-egg-deploy.sh "$(EGG_IMAGE_TAG)"
 	@echo "Deployment complete"
 
 redeploy: build k3s-import deploy  ## Rebuild, re-import, and redeploy in one step
diff --git a/scripts/await-egg-deploy.sh b/scripts/await-egg-deploy.sh
new file mode 100755
index 0000000000..3ea51a70cf
--- /dev/null
+++ b/scripts/await-egg-deploy.sh
@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+#
+# await-egg-deploy.sh - Wait for the egg-system deployments to become
+# Available, failing fast with an actionable message when the cause is
+# an image tag that was never imported into k3s.
+#
+# The orchestrator/gateway manifests reference egg-*:<EGG_IMAGE_TAG>,
+# where EGG_IMAGE_TAG is `git describe --always` and so changes on every
+# commit, pull, rebase, or checkout. `make redeploy` builds, imports, and
+# deploys in one invocation so the tag is self-consistent — but a bare
+# `make deploy` after HEAD has moved references a tag whose images are
+# not in k3s's containerd, and the pods sit in ImagePullBackOff.
+#
+# Without this guard that surfaces only as a bare 120s `kubectl wait`
+# timeout ("error: timed out waiting for the condition"). Here we detect
+# the ImagePullBackOff within seconds and point at the fix.
+#
+set -euo pipefail
+
+NS="egg-system"
+TAG="${1:-unknown}"
+TIMEOUT="${2:-180}"
+DEPLOYMENTS=(orchestrator gateway)
+
+deadline=$(( $(date +%s) + TIMEOUT ))
+
+while :; do
+  # Success: every deployment reports Available=True.
+  all_available=1
+  for d in "${DEPLOYMENTS[@]}"; do
+    avail=$(kubectl -n "$NS" get deployment "$d" \
+      -o jsonpath='{.status.conditions[?(@.type=="Available")].status}' 2>/dev/null || echo "")
+    [ "$avail" = "True" ] || all_available=0
+  done
+  if [ "$all_available" -eq 1 ]; then
+    echo "All egg-system deployments are Available."
+    exit 0
+  fi
+
+  # Fast-fail: a pod can't pull its image. Almost always tag drift —
+  # HEAD moved since the last build+import, so `make deploy` references
+  # egg-*:$TAG which was never imported into k3s.
+  if kubectl -n "$NS" get pods \
+      -o jsonpath='{range .items[*]}{range .status.containerStatuses[*]}{.state.waiting.reason}{"\n"}{end}{end}' \
+      2>/dev/null | grep -qE 'ImagePullBackOff|ErrImagePull'; then
+    echo "ERROR: egg-system pods cannot pull image tag '${TAG}' — it is not in k3s." >&2
+    echo "       A commit, pull, or rebase since your last build moved EGG_IMAGE_TAG." >&2
+    echo "       'make deploy' alone only deploys; run 'make redeploy' to rebuild +" >&2
+    echo "       re-import + deploy on the current tag." >&2
+    exit 1
+  fi
+
+  if [ "$(date +%s)" -ge "$deadline" ]; then
+    echo "ERROR: timed out after ${TIMEOUT}s waiting for egg-system deployments." >&2
+    echo "       Current pod state:" >&2
+    kubectl -n "$NS" get pods >&2 || true
+    exit 1
+  fi
+
+  sleep 3
+done

From d5dcfbc2b423e0f7366fbf9ff50874a79d2f448a Mon Sep 17 00:00:00 2001
From: James Wiesebron <jameswiesebron@khanacademy.org>
Date: Thu, 14 May 2026 22:42:37 -0700
Subject: [PATCH 7/8] await-egg-deploy: surface kubectl errors fast, fail loud
 on missing arg

Two non-blocking notes from the re-review on PR #2705:

1. Distinguish NotFound (transient, expected during early rollout) from
   real kubectl errors (auth, connection, RBAC). The previous
   `2>/dev/null || echo ""` swallowed everything, so a broken
   kubeconfig would poll silently for the full timeout before dumping
   an empty pod list. Now real errors surface within the first tick.

2. Replace the silent `TAG="${1:-unknown}"` fallback with `${1:?usage:
   ...}` so hand-running the script without an argument fails loud
   with a usage message, instead of producing a misleading
   "tag 'unknown'" diagnosis.

Skipped: the third note (a caveat for hypothetical remote-registry
topologies). Accurate for egg's local-k3s + `k3s ctr images import`
setup today; revisit if/when that path actually lands.
---
 scripts/await-egg-deploy.sh | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/scripts/await-egg-deploy.sh b/scripts/await-egg-deploy.sh
index 3ea51a70cf..d30007e673 100755
--- a/scripts/await-egg-deploy.sh
+++ b/scripts/await-egg-deploy.sh
@@ -18,7 +18,8 @@
 set -euo pipefail
 
 NS="egg-system"
-TAG="${1:-unknown}"
+: "${1:?usage: $0 <egg-image-tag> [timeout-seconds]}"
+TAG="$1"
 TIMEOUT="${2:-180}"
 DEPLOYMENTS=(orchestrator gateway)
 
@@ -28,9 +29,18 @@ while :; do
   # Success: every deployment reports Available=True.
   all_available=1
   for d in "${DEPLOYMENTS[@]}"; do
-    avail=$(kubectl -n "$NS" get deployment "$d" \
-      -o jsonpath='{.status.conditions[?(@.type=="Available")].status}' 2>/dev/null || echo "")
-    [ "$avail" = "True" ] || all_available=0
+    rc=0
+    out=$(kubectl -n "$NS" get deployment "$d" \
+      -o jsonpath='{.status.conditions[?(@.type=="Available")].status}' 2>&1) || rc=$?
+    if [ "$rc" -ne 0 ] && ! grep -q 'NotFound' <<<"$out"; then
+      # Real kubectl error (auth, connection, RBAC) — surface
+      # immediately rather than polling silently for the full timeout.
+      # NotFound is the expected "not yet observed" state during early
+      # rollout and falls through to the not-Available branch below.
+      echo "ERROR: kubectl get deployment $d failed: $out" >&2
+      exit 1
+    fi
+    [ "$out" = "True" ] || all_available=0
   done
   if [ "$all_available" -eq 1 ]; then
     echo "All egg-system deployments are Available."

From f19e59bf4eef5dbdb74fe9eda3e976ec1127b0b6 Mon Sep 17 00:00:00 2001
From: "egg-reviewer[bot]"
 <261018737+egg-reviewer[bot]@users.noreply.github.com>
Date: Fri, 15 May 2026 06:01:48 +0000
Subject: [PATCH 8/8] await-egg-deploy: capture kubectl stderr separately from
 stdout
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Previously the deployment-availability check captured kubectl's stderr
into the same variable as its stdout (`2>&1`) so the NotFound-vs-real-
error gate could grep it. That made the downstream
`[ "$out" = "True" ]` equality sensitive to any stderr line kubectl
might emit on a successful call — admission webhook deprecation
warnings, API-version advisories, near-expiry token notices, etc.
None of those fire on egg's current local-k3s, but the channel is
shared by design, so a future cluster could silently fail the equality
and poll until the 180s timeout.

Redirect stderr to a temp file instead, slurp it into $err for the
gate, and keep $out strictly equal to the jsonpath value. trap rms
the file on EXIT.
---
 scripts/await-egg-deploy.sh | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/scripts/await-egg-deploy.sh b/scripts/await-egg-deploy.sh
index d30007e673..64ea0d5d00 100755
--- a/scripts/await-egg-deploy.sh
+++ b/scripts/await-egg-deploy.sh
@@ -23,6 +23,15 @@ TAG="$1"
 TIMEOUT="${2:-180}"
 DEPLOYMENTS=(orchestrator gateway)
 
+# Keep kubectl stderr off of stdout so the success-path jsonpath value in
+# $out is strictly equal to the queried field. If a future cluster ever
+# emits a stderr warning on a successful call (admission webhook
+# deprecation, API-version advisory, token-near-expiry), merging it into
+# $out would silently fail the [ "$out" = "True" ] equality and poll
+# until timeout.
+err_file=$(mktemp)
+trap 'rm -f "$err_file"' EXIT
+
 deadline=$(( $(date +%s) + TIMEOUT ))
 
 while :; do
@@ -31,13 +40,14 @@ while :; do
   for d in "${DEPLOYMENTS[@]}"; do
     rc=0
     out=$(kubectl -n "$NS" get deployment "$d" \
-      -o jsonpath='{.status.conditions[?(@.type=="Available")].status}' 2>&1) || rc=$?
-    if [ "$rc" -ne 0 ] && ! grep -q 'NotFound' <<<"$out"; then
+      -o jsonpath='{.status.conditions[?(@.type=="Available")].status}' 2>"$err_file") || rc=$?
+    err=$(<"$err_file")
+    if [ "$rc" -ne 0 ] && ! grep -q 'NotFound' <<<"$err"; then
       # Real kubectl error (auth, connection, RBAC) — surface
       # immediately rather than polling silently for the full timeout.
       # NotFound is the expected "not yet observed" state during early
       # rollout and falls through to the not-Available branch below.
-      echo "ERROR: kubectl get deployment $d failed: $out" >&2
+      echo "ERROR: kubectl get deployment $d failed: $err" >&2
       exit 1
     fi
     [ "$out" = "True" ] || all_available=0