diff --git a/.dockerignore b/.dockerignore index 325dfa81db..fde5fe82c7 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,6 +4,28 @@ .git **/.git/ +# Claude Code worktrees and egg session state (issue #2999). Multi-GB on a +# live dev checkout, and `sandbox/Dockerfile` Stage 3 does +# `COPY . /opt/egg-runtime/` — without these exclusions every source edit +# re-ingests gigabytes of worktree/session content into a fresh image layer, +# which then churns through the docker store, the k3s transfer, and containerd. +.claude/ +.egg-state/ + +# Tool caches — same `COPY . /opt/egg-runtime/` bloat as above +**/.mypy_cache/ +**/.pytest_cache/ +**/.ruff_cache/ +**/.hypothesis/ + +# Sandbox build context — generated by scripts/prepare-sandbox-build-context.py, +# only used by sandbox/Dockerfile (`COPY repo-deps/ /tmp/repo-deps/`). Excluding +# it here keeps the gateway/orchestrator builds from shipping it as part of +# their build context. The sandbox build re-includes it via the per-Dockerfile +# `sandbox/Dockerfile.dockerignore` override — keep these two files in sync +# (drift mechanically enforced by tests/scripts/test_dockerignore_drift.py). +repo-deps/ + # Mobile app dependencies and build artifacts **/mobile/app/ios/Pods/ **/mobile/app/ios/build/ diff --git a/.github/workflows/test-integration.yml b/.github/workflows/test-integration.yml index a750f141ee..07c82fbdb7 100644 --- a/.github/workflows/test-integration.yml +++ b/.github/workflows/test-integration.yml @@ -17,6 +17,16 @@ jobs: # whether this workflow is invoked via `uses:` or via # `workflow_dispatch`. timeout-minutes: 30 + env: + # Keep CI on the docker-save/ctr-import publish path (the inline + # "Import images into k3s" step below). The localhost registry + + # registries.yaml flow is a local-dev optimization (issue #2999) and + # would be pure setup overhead on an ephemeral runner. Empty value = + # bare image names in `make build` tags and `make deploy` rewrites. + EGG_IMAGE_REGISTRY: "" + # Sequential image builds: the hosted runner builds every stage cold, + # and four concurrent heavyweight builds risk memory/disk flake. + BUILD_JOBS: "1" steps: - uses: actions/checkout@v4 diff --git a/Makefile b/Makefile index 672c97c8c5..bc47544744 100644 --- a/Makefile +++ b/Makefile @@ -27,14 +27,55 @@ PYTHON = $(if $(wildcard $(VENV_BIN)/python),$(VENV_BIN)/python,python3) # Falls back to "latest" outside a git checkout. EGG_IMAGE_TAG := $(shell git describe --always --dirty 2>/dev/null || echo latest) +# Local image registry for the k3s deploy flow (issue #2999). When set, +# `make redeploy` publishes images by `docker push`-ing here and lets the +# cluster pull them back — both sides are layer-aware, so a typical +# code-only rebuild moves tens of MB instead of the full ~multi-GB sandbox +# image that `docker save` + `ctr import` always re-serialize. One-time +# host setup: `make registry-setup`. Set EGG_IMAGE_REGISTRY= (empty) to +# fall back to the save+import flow (`make k3s-import`) — CI does this so +# its inline import path keeps working without a registry. +EGG_IMAGE_REGISTRY ?= localhost:5000 +# "localhost:5000/" or "" — spliced ahead of image names in build tags and +# in the manifest rewrites in `deploy`. +EGG_IMAGE_PREFIX := $(if $(EGG_IMAGE_REGISTRY),$(EGG_IMAGE_REGISTRY)/,) + +# The full egg image set, and the subset published via the registry. The +# sandbox image is EXCLUDED from the registry subset by default: it bakes in +# private repo content (node_modules/.venv/anything repositories.yaml +# build_commands produce), so it never goes near a registry — even the +# loopback-only local one — unless the operator opts in by adding +# egg-sandbox to EGG_REGISTRY_IMAGES. Excluded images publish through the +# save+import path instead (slower for the big sandbox image, but entirely +# store-to-store on this host). push-egg-images.sh independently refuses any +# non-loopback registry, so opting in still cannot publish off-host. +EGG_ALL_IMAGES := egg-gateway egg-orchestrator egg-sandbox egg-litellm +EGG_REGISTRY_IMAGES ?= egg-gateway egg-orchestrator egg-litellm +# Images the registry path does NOT cover (imported via k3s-import instead). +EGG_IMPORT_IMAGES := $(filter-out $(EGG_REGISTRY_IMAGES),$(EGG_ALL_IMAGES)) +# Per-image manifest prefix: registry-qualified only when registry mode is on +# AND the image is in the registry subset. +reg_prefix = $(if $(filter $(1),$(EGG_REGISTRY_IMAGES)),$(EGG_IMAGE_PREFIX),) + +# Parallel image builds (issue #2999): the four images are independent, so +# build them concurrently. BUILD_JOBS=1 restores sequential builds — CI sets +# this because hosted runners build every stage cold, and four concurrent +# heavyweight builds risk memory/disk flake there. --output-sync (buffer +# each sub-build's output so the BuildKit progress UIs don't interleave) +# needs GNU make 4+; on 3.x (stock macOS make) output interleaves but the +# builds still work. +BUILD_JOBS ?= 4 +BUILD_OUTPUT_SYNC := $(if $(filter 3.%,$(MAKE_VERSION)),,--output-sync=target) + .PHONY: help \ setup deps venv sync-venv-if-uv sandbox-deps install-linters check-linters \ lint lint-python lint-shell lint-yaml lint-docker lint-actions lint-custom \ test test-all test-record-good security \ test-integration test-security smoketest-long-poll \ lint-fix lint-python-fix lint-shell-fix lint-yaml-fix \ - build \ - k3s-setup k3s-secrets litellm-config routing-policy deploy redeploy k3s-teardown k3s-import sudo-keepalive \ + build build-gateway build-orchestrator build-sandbox build-litellm \ + k3s-setup k3s-secrets litellm-config routing-policy deploy redeploy k3s-teardown \ + k3s-import k3s-push k3s-publish registry-setup btrfs-reclaim sudo-keepalive \ check-egg-images-present # Default target @@ -76,9 +117,12 @@ help: @echo "" @echo "Kubernetes (k3s):" @echo " make k3s-setup - Install k3s with Cilium CNI" + @echo " make registry-setup - One-time: local image registry + k3s registries.yaml" @echo " make deploy - Deploy egg to k3s" - @echo " make redeploy - Rebuild, re-import, and redeploy in one step" - @echo " make k3s-import - Import built images into k3s" + @echo " make redeploy - Rebuild, publish images, and redeploy in one step" + @echo " make k3s-push - Push built images to the local registry" + @echo " make k3s-import - Import built images into k3s (no-registry fallback)" + @echo " make btrfs-reclaim - Reclaim btrfs over-allocated chunks (issue #2999)" @echo " make k3s-teardown - Remove k3s" # ============================================================================ @@ -460,18 +504,45 @@ lint-yaml-fix: sync-venv-if-uv # Build # ============================================================================ +# Tag set for one image: bare names always (k3s-import + local tooling), +# registry-qualified names only when registry mode is on AND the image is in +# the registry subset (what `k3s-push` pushes and `deploy` rewrites the +# manifests to). Non-subset images (the sandbox, by default) never even get +# a registry-qualified tag. +define image_tags +-t $(1):latest -t $(1):$(EGG_IMAGE_TAG) $(if $(call reg_prefix,$(1)),-t $(call reg_prefix,$(1))$(1):latest -t $(call reg_prefix,$(1))$(1):$(EGG_IMAGE_TAG)) +endef + build: sync-venv-if-uv @echo "==> Preparing sandbox build context from repositories.yaml..." @$(PYTHON) scripts/prepare-sandbox-build-context.py repo-deps - @echo "==> Building images with tag $(EGG_IMAGE_TAG)..." - @echo "==> Building gateway container..." - docker build -t egg-gateway:latest -t egg-gateway:$(EGG_IMAGE_TAG) -f gateway/Dockerfile . - @echo "==> Building orchestrator container..." - docker build -t egg-orchestrator:latest -t egg-orchestrator:$(EGG_IMAGE_TAG) -f orchestrator/Dockerfile . - @echo "==> Building sandbox container..." - docker build -t egg-sandbox:latest -t egg-sandbox:$(EGG_IMAGE_TAG) -f sandbox/Dockerfile . - @echo "==> Building litellm container (stock LiteLLM + egg cache patches)..." - docker build -t egg-litellm:latest -t egg-litellm:$(EGG_IMAGE_TAG) -f config/litellm/Dockerfile config/litellm + @echo "==> Building images with tag $(EGG_IMAGE_TAG) ($(BUILD_JOBS) parallel jobs)..." + @$(MAKE) --no-print-directory -j$(BUILD_JOBS) $(BUILD_OUTPUT_SYNC) \ + build-gateway build-orchestrator build-sandbox build-litellm + +# Per-image sub-targets so `build` can run them under -j. They assume the +# repo-deps/ build context has been prepared (the `build` target does that +# first, sequentially) — run `build`, not these, unless you know repo-deps/ +# is fresh. +# +# DOCKER_BUILDKIT=1 is required so BuildKit honors the per-Dockerfile +# `.dockerignore` lookup (see sandbox/Dockerfile.dockerignore). +# The legacy builder reads only the root .dockerignore — under it the +# sandbox build's `COPY repo-deps/ /tmp/repo-deps/` would fail because +# repo-deps/ is excluded at the root. Docker 23+ defaults to BuildKit; +# pinning the env var keeps older Docker (18.09–22.x, which supports +# BuildKit but doesn't default to it) on the same path. +build-gateway: + DOCKER_BUILDKIT=1 docker build $(call image_tags,egg-gateway) -f gateway/Dockerfile . + +build-orchestrator: + DOCKER_BUILDKIT=1 docker build $(call image_tags,egg-orchestrator) -f orchestrator/Dockerfile . + +build-sandbox: + DOCKER_BUILDKIT=1 docker build $(call image_tags,egg-sandbox) -f sandbox/Dockerfile . + +build-litellm: + DOCKER_BUILDKIT=1 docker build $(call image_tags,egg-litellm) -f config/litellm/Dockerfile config/litellm # ============================================================================ # Kubernetes (k3s) targets @@ -497,6 +568,9 @@ k3s-setup: ## Install k3s with Cilium CNI echo "Waiting for k3s node to be ready..." && \ kubectl wait --for=condition=Ready node --all --timeout=120s && \ scripts/install-metrics-server.sh + @if [ -n "$(EGG_IMAGE_REGISTRY)" ]; then \ + $(MAKE) --no-print-directory registry-setup; \ + fi @echo "k3s cluster ready" k3s-secrets: ## Create gateway secrets from ~/.config/egg/ @@ -593,7 +667,7 @@ routing-policy: ## Apply host-side gateway routing policy from ~/.config/egg/ro @echo " running gateway pod in ~60s; the gateway re-reads it on the next request." check-egg-images-present: - @scripts/check-egg-images-present.sh "$(EGG_IMAGE_TAG)" + @scripts/check-egg-images-present.sh "$(EGG_IMAGE_TAG)" "$(EGG_IMAGE_REGISTRY)" $(EGG_REGISTRY_IMAGES) # Cluster-mutating steps (k3s-secrets, kubectl apply) are invoked from the # recipe body so the ordering survives `make -j`: two prerequisites of the @@ -625,19 +699,21 @@ deploy: sudo-keepalive check-egg-images-present ## Deploy egg to k3s kubectl kustomize k8s/overlays/local/ | \ envsubst '$$EGG_HOST_HOME $$EGG_HOST_REPO_MAP' | \ sed -E "/name: EGG_HOST_REPO_MAP$$/{N;s|^(\s*- name: EGG_HOST_REPO_MAP\s*\n\s*value: )(\{.*\})$$|\1'\2'|}" | \ - sed -e "s|egg-orchestrator:latest|egg-orchestrator:$(EGG_IMAGE_TAG)|g" \ - -e "s|egg-gateway:latest|egg-gateway:$(EGG_IMAGE_TAG)|g" \ - -e "s|egg-sandbox:latest|egg-sandbox:$(EGG_IMAGE_TAG)|g" \ - -e "s|egg-litellm:latest|egg-litellm:$(EGG_IMAGE_TAG)|g" | \ + sed -e "s|egg-orchestrator:latest|$(call reg_prefix,egg-orchestrator)egg-orchestrator:$(EGG_IMAGE_TAG)|g" \ + -e "s|egg-gateway:latest|$(call reg_prefix,egg-gateway)egg-gateway:$(EGG_IMAGE_TAG)|g" \ + -e "s|egg-sandbox:latest|$(call reg_prefix,egg-sandbox)egg-sandbox:$(EGG_IMAGE_TAG)|g" \ + -e "s|egg-litellm:latest|$(call reg_prefix,egg-litellm)egg-litellm:$(EGG_IMAGE_TAG)|g" | \ kubectl apply -f - && \ scripts/clear-stuck-egg-pods.sh && \ scripts/await-egg-deploy.sh "$(EGG_IMAGE_TAG)" - @# Rollout confirmed on $(EGG_IMAGE_TAG): drop older egg image tags from - @# containerd so it does not accumulate a ~12 GB sandbox image per deployed - @# commit and push the root fs over kubelet's image-GC threshold (which would - @# evict the next redeploy's freshly-imported, not-yet-referenced images - @# mid-run). Best-effort -- a reap hiccup must not fail an otherwise-green deploy. - @scripts/reap-stale-egg-images.sh "$(EGG_IMAGE_TAG)" || true + @# Rollout confirmed on $(EGG_IMAGE_TAG): reap stale egg images from + @# containerd, the docker store/BuildKit cache, and the local registry so + @# none of them accumulates a ~10 GB sandbox image per deployed commit and + @# pushes the root fs over kubelet's image-GC threshold. On btrfs hosts it + @# also warns/auto-balances when chunk over-allocation runs the unallocated + @# pool low (issue #2999). Best-effort -- a reap hiccup must not fail an + @# otherwise-green deploy. + @scripts/reap-stale-egg-images.sh "$(EGG_IMAGE_TAG)" "$(EGG_IMAGE_REGISTRY)" $(EGG_REGISTRY_IMAGES) || true @# routing-policy.yaml (issue #2987) was already bundled by the @# k3s-secrets call at the top of this target; no separate apply needed @# here. `make routing-policy` is the standalone hot-reload path between @@ -645,7 +721,38 @@ deploy: sudo-keepalive check-egg-images-present ## Deploy egg to k3s @$(MAKE) --no-print-directory litellm-config @echo "Deployment complete" -redeploy: sudo-keepalive build k3s-import deploy ## Rebuild, re-import, and redeploy in one step +redeploy: sudo-keepalive build k3s-publish deploy ## Rebuild, publish images, and redeploy in one step + +# Publish dispatch (issue #2999): in registry mode, push the registry subset +# (layer-incremental — the fast path) and save+import the rest (the sandbox, +# unless opted in via EGG_REGISTRY_IMAGES); without a registry — or with an +# empty registry subset (EGG_REGISTRY_IMAGES="") — save+import everything. +k3s-publish: sudo-keepalive + @if [ -n "$(EGG_IMAGE_REGISTRY)" ] && [ -n "$(strip $(EGG_REGISTRY_IMAGES))" ]; then \ + $(MAKE) --no-print-directory k3s-push; \ + if [ -n "$(strip $(EGG_IMPORT_IMAGES))" ]; then \ + $(MAKE) --no-print-directory k3s-import K3S_IMPORT_IMAGES="$(EGG_IMPORT_IMAGES)"; \ + fi; \ + else \ + $(MAKE) --no-print-directory k3s-import; \ + fi + +# Push the registry-subset images to the local registry, then pre-pull them +# into k3s's containerd. The pre-pull (sudo, hence the sudo-keepalive +# prerequisite) keeps pod starts instant and lets reap-stale-egg-images.sh's +# safety gate see the new refs before anything references them. +k3s-push: sudo-keepalive ## Push registry-subset images to the local registry + @scripts/push-egg-images.sh "$(EGG_IMAGE_TAG)" "$(EGG_IMAGE_REGISTRY)" $(EGG_REGISTRY_IMAGES) + +registry-setup: ## One-time: run the local image registry + point k3s at it + @scripts/setup-local-registry.sh "$(EGG_IMAGE_REGISTRY)" + +# Manual escape hatch for #2999's root disease: btrfs data-chunk +# over-allocation from image churn. The post-deploy reap warns when this is +# needed and auto-runs it when critically low; run it by hand any time +# DiskPressure / "images not in k3s" shows up while `df -h` claims space. +btrfs-reclaim: ## Reclaim btrfs over-allocated data chunks (issue #2999) + @scripts/btrfs-reclaim.sh # Prompt for the sudo password immediately so `make redeploy` can be left # unattended through the long `build` step. A detached background loop refreshes @@ -691,8 +798,17 @@ sudo-keepalive: # non-egg content -- no tool in this repo does so, and an out-of-band retag # would only mismatch if :$(EGG_IMAGE_TAG) is also absent (the inner grep # guard skips when it is already present). +# NOTE (#2999): k3s-import is the save+import publish path. The default +# `make redeploy` flow uses it only for the images excluded from the registry +# subset (the sandbox, which must not be pushed to any registry by default — +# see EGG_REGISTRY_IMAGES above); with EGG_IMAGE_REGISTRY= (empty) it covers +# everything, e.g. for CI. K3S_IMPORT_IMAGES narrows the image set +# (k3s-publish passes the registry-subset complement). It deals in BARE +# image names (egg-sandbox:), matching what `deploy` writes into the +# manifests for non-registry images. +K3S_IMPORT_IMAGES ?= $(EGG_ALL_IMAGES) k3s-import: SHELL := /bin/bash -k3s-import: sudo-keepalive ## Import built images into k3s +k3s-import: sudo-keepalive ## Import built images into k3s (registry-excluded set) @set -euo pipefail; \ tmp=$$(mktemp -d -p /var/tmp egg-k3s-import.XXXXXX); \ trap 'rm -rf "$$tmp"' EXIT; \ @@ -700,7 +816,7 @@ k3s-import: sudo-keepalive ## Import built images into k3s if [ "$(EGG_IMAGE_TAG)" != "latest" ]; then tags="$$tags $(EGG_IMAGE_TAG)"; fi; \ id_dir="$${XDG_CACHE_HOME:-$$HOME/.cache}/egg/k3s-import-ids"; \ mkdir -p "$$id_dir"; \ - for image in egg-gateway egg-orchestrator egg-sandbox egg-litellm; do \ + for image in $(K3S_IMPORT_IMAGES); do \ cur_id=$$(docker image inspect "$$image:$(EGG_IMAGE_TAG)" --format '{{.Id}}'); \ marker="$$id_dir/$$image.id"; \ prev_id=$$(cat "$$marker" 2>/dev/null || true); \ @@ -715,14 +831,13 @@ k3s-import: sudo-keepalive ## Import built images into k3s fi; \ done; \ else \ - for tag in $$tags; do \ - img="$$image:$$tag"; \ - f="$$tmp/$${img//[:\/]/_}.tar"; \ - echo ">>> importing $$img"; \ - docker save "$$img" -o "$$f"; \ - sudo k3s ctr images import "$$f"; \ - rm -f "$$f"; \ - done; \ + refs=""; \ + for tag in $$tags; do refs="$$refs $$image:$$tag"; done; \ + f="$$tmp/$$image.tar"; \ + echo ">>> importing$$refs (one tarball: tags share all layers)"; \ + docker save $$refs -o "$$f"; \ + sudo k3s ctr images import "$$f"; \ + rm -f "$$f"; \ printf '%s\n' "$$cur_id" > "$$marker.tmp" && mv -f "$$marker.tmp" "$$marker"; \ present=$$(sudo k3s ctr images list -q); \ fi; \ diff --git a/config/README.md b/config/README.md index a8267201ad..88b7f8e8e5 100644 --- a/config/README.md +++ b/config/README.md @@ -207,7 +207,7 @@ Each `build_commands` entry has: - `watch_files`: Files that trigger a rebuild when changed. These are copied into the Docker build context so Docker layer caching invalidates correctly — only a change to these files triggers a dependency rebuild. - `commands`: Shell commands to run during the image build (e.g., `npm ci`, `pip install`, `make deps`). Commands run as root in a directory seeded with the watch files. - `persist_dirs`: Directories (relative to repo root) to preserve from the build context into the Docker image. After `commands` run, these directories are copied to `/opt/prebuilt-deps//` and restored into the mounted repo at container startup by `entrypoint.py`. Use this for local dependencies like `node_modules` that would otherwise be lost when the build context is cleaned up. -- `persist_system_dirs`: Absolute-path system directories to preserve from the build stage into the final image. After `commands` run, these directories are copied to `/opt/prebuilt-deps/__egg_system_dirs__//` and the Dockerfile restores them to their original absolute locations. Use this for system-level tool installations that land outside the repo directory (e.g., `/usr/local/go`, `/usr/local/node`). Top-level system paths (`/`, `/etc`, `/usr`, `/var`, etc.) and all paths under `/proc`, `/sys`, `/dev`, `/run`, `/boot` are blocked. +- `persist_system_dirs`: Absolute-path system directories to preserve from the build stage into the final image. After `commands` run, these directories are copied to `/opt/egg-system-dirs//` and the Dockerfile restores them to their original absolute locations. Use this for system-level tool installations that land outside the repo directory (e.g., `/usr/local/go`, `/usr/local/node`). Top-level system paths (`/`, `/etc`, `/usr`, `/var`, etc.) and all paths under `/proc`, `/sys`, `/dev`, `/run`, `/boot` are blocked. **Fail-fast contract (since #2087):** A non-zero exit from any command, a missing watch-files directory, or a `persist_dirs` / `persist_system_dirs` entry that doesn't exist after the commands run all abort the image build. Earlier behavior printed a warning and continued, which silently produced empty `/opt/prebuilt-deps//` trees. Path-traversal rejection in `persist_dirs` remains warn-and-skip — it's a security control, not a misconfiguration we want to crash on. @@ -216,8 +216,8 @@ Each `build_commands` entry has: 2. The `COPY repo-deps/` layer in the Dockerfile keys on watch-file contents, so a change to (e.g.) `package-lock.json` invalidates the dependency layer on the next `make build` 3. During `docker build`, the `docker-setup.py` script reads both `build_commands` and `extra_packages` from `manifest.json` (since `repositories.yaml` is unavailable in the build context) and executes them in order 4. All repos' dependencies are installed into the **same image** — no per-repo images -5. After commands run, `persist_build_dirs()` copies `persist_dirs` entries to `/opt/prebuilt-deps//` and `persist_system_dirs` entries to `/opt/prebuilt-deps/__egg_system_dirs__//` inside the image -6. The Dockerfile copies `__egg_system_dirs__` back to `/` so system tools are available at their original absolute paths +5. After commands run, `persist_build_dirs()` copies `persist_dirs` entries to `/opt/prebuilt-deps//` and `persist_system_dirs` entries to `/opt/egg-system-dirs//` inside the image (separate bases so the final stage stores the system dirs once, not twice — issue #2999) +6. The Dockerfile copies `/opt/egg-system-dirs/` back to `/` so system tools are available at their original absolute paths 7. At container startup, `restore_prebuilt_deps()` in `entrypoint.py` copies `persist_dirs` directories into the mounted repo (skipping files that already exist) **Key properties:** diff --git a/docs/architecture/orchestrator.md b/docs/architecture/orchestrator.md index ef64749efa..2f45ee9c0a 100644 --- a/docs/architecture/orchestrator.md +++ b/docs/architecture/orchestrator.md @@ -1245,7 +1245,7 @@ event-handler contract (task-3-4) only reaches the agent pod after: ```bash make build # rebuild egg-sandbox / egg-orchestrator / egg-gateway / egg-litellm -make k3s-import # import rebuilt images into k3s +make k3s-push # publish rebuilt images (or k3s-import without the local registry) make deploy # roll out deployments in egg-system ``` diff --git a/docs/guides/deployment.md b/docs/guides/deployment.md index c3b7c39fb0..7a71edfc4f 100644 --- a/docs/guides/deployment.md +++ b/docs/guides/deployment.md @@ -275,8 +275,9 @@ Pre-built `egg-sandbox` images on GHCR are built without a host to populate `.venv`, `npm ci` to populate `node_modules`) are **not** applied to the published image. Operators relying on prebuilt deps — including the project's own `.venv` for `make test` / `make lint` — should -build the image locally with `make build` and import it via -`make k3s-import` rather than pulling from GHCR. `make build` runs +build the image locally with `make build` and publish it via +`make k3s-push` (or `make k3s-import` on hosts without the local +registry) rather than pulling from GHCR. `make build` runs `scripts/prepare-sandbox-build-context.py` to populate `repo-deps/` from your `repositories.yaml` before the Docker build, so per-repo deps are included (see #2499). @@ -459,7 +460,7 @@ Fix: rebuild and re-import the sandbox image. The legacy `egg --reset` shortcut ```bash make build # rebuild egg-sandbox / egg-orchestrator / egg-gateway / egg-litellm images -make k3s-import # import rebuilt images into k3s +make k3s-push # publish rebuilt images (or k3s-import without the local registry) make deploy # roll out deployments in egg-system ``` diff --git a/docs/guides/local-quickstart.md b/docs/guides/local-quickstart.md index dde1d6dd76..10417118c8 100644 --- a/docs/guides/local-quickstart.md +++ b/docs/guides/local-quickstart.md @@ -70,15 +70,20 @@ Setting `auth_mode: user` tells the gateway to use `GITHUB_USER_TOKEN` for git/g ## 3. Build and deploy ```bash +make registry-setup # one-time: local image registry + k3s registries.yaml make build # build gateway, orchestrator, and sandbox images -make k3s-import # import images into k3s containerd store +make k3s-push # push images to the local registry + pre-pull into k3s make k3s-secrets # create k8s Secrets from ~/.config/egg/ make deploy # deploy gateway + orchestrator to k3s (idempotent) kubectl get pods -n egg-system # verify pods are running egg --public # start sandbox session ``` -`make build` builds the Docker images. `make k3s-import` imports them into k3s's containerd image store (without this, pods will get `ImagePullBackOff`). `make deploy` applies the Kustomize manifests — it is idempotent and can be re-run after code changes to update the running deployment. `make deploy` also performs a pre-flight check: it aborts before touching the cluster if the images for the current tag are not yet in k3s, directing you to run `make redeploy` instead. +`make registry-setup` is a one-time host step (also run by `make k3s-setup`): it starts a `registry:2` container **bound to 127.0.0.1 only** (nothing is published off-host) and points k3s's containerd at it via `/etc/rancher/k3s/registries.yaml`. `make build` builds the Docker images. `make k3s-push` publishes the registry-subset images through the registry — `docker push` and the containerd pull are both layer-aware, so after the first publish a code-only rebuild moves tens of MB instead of full images (issue #2999). `make deploy` applies the Kustomize manifests — it is idempotent and can be re-run after code changes to update the running deployment. `make deploy` also performs a pre-flight check: it aborts before touching the cluster if the images for the current tag were never published, directing you to run `make redeploy` instead. + +By default the registry subset is `egg-gateway egg-orchestrator egg-litellm` (`EGG_REGISTRY_IMAGES`). The **sandbox image is excluded**: it bakes in private repo content (prebuilt `node_modules`/`.venv` from `repositories.yaml` build commands), so it is never pushed to any registry — even the loopback one — unless you opt in by adding `egg-sandbox` to `EGG_REGISTRY_IMAGES`; it publishes via `docker save` + `ctr import` instead (store-to-store on this host, no registry involved). Opting in makes sandbox publishes layer-incremental too, and `push-egg-images.sh` hard-refuses non-loopback registries either way. + +In practice you rarely run these individually — `make redeploy` chains build → publish → deploy. To run without the local registry entirely (e.g. an ephemeral CI host), set `EGG_IMAGE_REGISTRY=` (empty); every image then goes through `make k3s-import`. ## 4. Using the SDLC pipeline diff --git a/docs/reference/agent-wait-patterns.md b/docs/reference/agent-wait-patterns.md index 730ddd256d..43100cfd35 100644 --- a/docs/reference/agent-wait-patterns.md +++ b/docs/reference/agent-wait-patterns.md @@ -1654,7 +1654,7 @@ image is rebuilt and pods are restarted: ```bash make build # rebuild egg-sandbox / egg-orchestrator / egg-gateway / egg-litellm -make k3s-import # import rebuilt images into k3s +make k3s-push # publish rebuilt images (or k3s-import without the local registry) make deploy # roll out deployments in egg-system ``` diff --git a/sandbox/Dockerfile b/sandbox/Dockerfile index 6f29c1b4c8..75055cfa91 100644 --- a/sandbox/Dockerfile +++ b/sandbox/Dockerfile @@ -53,13 +53,16 @@ COPY repo-deps/ /tmp/repo-deps/ COPY sandbox/docker-setup.py /tmp/docker-setup.py # Run docker-setup.py: installs extra packages, runs build commands, -# persists output dirs to /opt/prebuilt-deps/ +# persists repo output dirs to /opt/prebuilt-deps/ and system-level installs +# (e.g. /usr/local/go) to /opt/egg-system-dirs/. The trailing mkdirs make +# both paths exist even when nothing was persisted, so the final stage's +# COPY --from of each cannot fail on a missing source. RUN chmod +x /tmp/docker-setup.py && \ apt-get update && \ python3 /tmp/docker-setup.py && \ rm /tmp/docker-setup.py && \ rm -rf /tmp/repo-deps && \ - mkdir -p /opt/prebuilt-deps + mkdir -p /opt/prebuilt-deps /opt/egg-system-dirs # ============================================================================= # Stage 2: base — system packages, Python 3.14, pip deps, Claude tools @@ -280,11 +283,14 @@ RUN if [ -f /opt/prebuilt-deps/extra-packages-apt.txt ] && [ -s /opt/prebuilt-de # Restore system-level directories installed by build_commands in Stage 1. # These are absolute-path installations (e.g., /usr/local/go, /usr/local/node) -# that were persisted to /opt/prebuilt-deps/__egg_system_dirs__/ by docker-setup.py. -# Copy the entire tree back to / so paths are restored to their original locations. -RUN if [ -d /opt/prebuilt-deps/__egg_system_dirs__ ]; then \ - cp -a /opt/prebuilt-deps/__egg_system_dirs__/. /; \ - fi +# that were persisted to /opt/egg-system-dirs/ by docker-setup.py. +# COPY straight from Stage 1 to / so paths land at their original locations. +# This must be a COPY --from, not a `cp` of an already-COPYed tree: copying +# a directory the image already contains stores its multi-GB content TWICE +# (once in the source layer, once in the cp layer) — the pre-#2999 layout +# did exactly that via /opt/prebuilt-deps/__egg_system_dirs__ and cost ~2 GB +# of image size. +COPY --from=repo-deps /opt/egg-system-dirs/ / # Copy sandbox runtime scripts and tools to /opt/egg-runtime # This provides container-resident executables available in PATH diff --git a/sandbox/Dockerfile.dockerignore b/sandbox/Dockerfile.dockerignore new file mode 100644 index 0000000000..f2c5985c71 --- /dev/null +++ b/sandbox/Dockerfile.dockerignore @@ -0,0 +1,67 @@ +# Per-Dockerfile .dockerignore for sandbox/Dockerfile (BuildKit feature). +# When present at .dockerignore, BuildKit uses it instead of the +# root .dockerignore — there is no merging. Mirror the root .dockerignore +# here MINUS the `repo-deps/` exclusion: the sandbox build (uniquely) needs +# repo-deps/ in its context for `COPY repo-deps/ /tmp/repo-deps/`. +# +# KEEP IN SYNC WITH the root .dockerignore: any new exclusion added there +# must be added here too, or the sandbox build context will silently bloat. +# Mechanically enforced by tests/scripts/test_dockerignore_drift.py. + +# Git directories (avoid permission issues and unnecessary context) +.git +**/.git/ + +# Claude Code worktrees and egg session state (issue #2999). Multi-GB on a +# live dev checkout, and `sandbox/Dockerfile` Stage 3 does +# `COPY . /opt/egg-runtime/` — without these exclusions every source edit +# re-ingests gigabytes of worktree/session content into a fresh image layer, +# which then churns through the docker store, the k3s transfer, and containerd. +.claude/ +.egg-state/ + +# Tool caches — same `COPY . /opt/egg-runtime/` bloat as above +**/.mypy_cache/ +**/.pytest_cache/ +**/.ruff_cache/ +**/.hypothesis/ + +# Mobile app dependencies and build artifacts +**/mobile/app/ios/Pods/ +**/mobile/app/ios/build/ +**/mobile/app/android/build/ +**/mobile/node_modules/ + +# Terraform vendored modules (contain nested .git directories) +**/.terraform/ + +# Node modules (can be reinstalled if needed) +**/node_modules/ + +# Python virtual environments +**/__pycache__/ +**/*.pyc +**/.venv/ +**/.virtualenv/ + +# Build artifacts +**/dist/ +**/build/ +**/*.egg-info/ + +# IDE and editor files +**/.vscode/ +**/.idea/ +**/*.swp +**/*.swo + +# OS files +**/.DS_Store +**/Thumbs.db + +# Logs +**/*.log + +# Temporary files +**/tmp/ +**/temp/ diff --git a/sandbox/README.md b/sandbox/README.md index 7f37a31213..0208af9a09 100644 --- a/sandbox/README.md +++ b/sandbox/README.md @@ -184,8 +184,8 @@ Per-repo `build_commands` in `repositories.yaml` allow project-specific dependen 1. `make build` invokes `scripts/prepare-sandbox-build-context.py`, which calls `populate_build_context()` (in `egg_lib/docker.py`) to copy each repo's `watch_files` from local paths into the build context at `repo-deps//` and write `repo-deps/manifest.json` 2. The Dockerfile `COPY repo-deps/` layer picks up these files — changes to watch files (e.g., `package-lock.json`) invalidate the Docker cache for this layer 3. `docker-setup.py` reads `build_commands` and `extra_packages` from `manifest.json` (since `repositories.yaml` is unavailable in the build context) and executes each repo's commands in its watch files directory -4. If `persist_dirs` is configured, `persist_build_dirs()` copies those directories (e.g., `node_modules`) from the build context to `/opt/prebuilt-deps//` in the image; if `persist_system_dirs` is configured, absolute-path system directories (e.g., `/usr/local/go`) are copied to `/opt/prebuilt-deps/__egg_system_dirs__//` -5. The Dockerfile restores `__egg_system_dirs__` to `/` so system tools are available at their original absolute paths in the final image +4. If `persist_dirs` is configured, `persist_build_dirs()` copies those directories (e.g., `node_modules`) from the build context to `/opt/prebuilt-deps//` in the image; if `persist_system_dirs` is configured, absolute-path system directories (e.g., `/usr/local/go`) are copied to `/opt/egg-system-dirs//` (a separate base from `/opt/prebuilt-deps` so the final stage doesn't store the multi-GB system dirs twice — issue #2999) +5. The Dockerfile `COPY --from`s `/opt/egg-system-dirs/` to `/` so system tools are available at their original absolute paths in the final image 6. At container startup, `restore_prebuilt_deps()` in `entrypoint.py` restores `persist_dirs` directories into the mounted repo, making them available without network access > **Cache invalidation is Docker's job.** The `COPY repo-deps/` layer keys on watch-file contents, so changing `package-lock.json` (or any declared watch file) invalidates the dependency layer on the next `make build`. There's no separate egg-side hash check; `make build` is always a no-op when nothing relevant changed and a full rebuild when something did. diff --git a/sandbox/docker-setup.py b/sandbox/docker-setup.py index 0f15e017be..b20b2b5915 100755 --- a/sandbox/docker-setup.py +++ b/sandbox/docker-setup.py @@ -372,6 +372,7 @@ def persist_build_dirs( build_commands: list[dict[str, Any]], repo_deps_base: Path = Path("/tmp/repo-deps"), prebuilt_base: Path = Path("/opt/prebuilt-deps"), + system_base: Path = Path("/opt/egg-system-dirs"), ) -> None: """Persist directories from build context into the Docker image. @@ -379,11 +380,20 @@ def persist_build_dirs( copied to a persistent location so they survive the /tmp/repo-deps cleanup. They are restored into mounted repos at container startup by entrypoint.py. + System-level absolute paths (persist_system_dirs) go to a SEPARATE base + (system_base, default /opt/egg-system-dirs) rather than under + prebuilt_base: the Dockerfile's final stage COPYs prebuilt_base verbatim + and restores system_base to ``/`` — keeping them apart means the multi-GB + system dirs land in the image exactly once instead of twice (issue #2999). + Args: build_commands: List of dicts with 'repo', 'commands', 'persist_dirs', and 'persist_system_dirs' keys. repo_deps_base: Base path for repo build contexts (default: /tmp/repo-deps). - prebuilt_base: Destination base for persisted directories (default: /opt/prebuilt-deps). + prebuilt_base: Destination base for persisted repo directories + (default: /opt/prebuilt-deps). + system_base: Destination base for persisted system directories + (default: /opt/egg-system-dirs). """ persist_count = 0 for entry in build_commands: @@ -427,9 +437,13 @@ def persist_build_dirs( DENIED_PREFIXES = ("/proc", "/sys", "/dev", "/run", "/boot") DENIED_EXACT = ("/", "/etc", "/bin", "/sbin", "/lib", "/lib64", "/usr", "/var") - # Also deny the prebuilt and repo-deps directories themselves to prevent - # self-referential copies that would bloat the image - denied_exact = DENIED_EXACT + (str(prebuilt_base), str(repo_deps_base)) + # Also deny the prebuilt/system/repo-deps directories themselves to + # prevent self-referential copies that would bloat the image + denied_exact = DENIED_EXACT + ( + str(prebuilt_base), + str(repo_deps_base), + str(system_base), + ) # Persist system-level directories (absolute paths like /usr/local/go) for entry in build_commands: @@ -438,7 +452,7 @@ def persist_build_dirs( if not system_dirs: continue - system_dest = prebuilt_base / "__egg_system_dirs__" + system_dest = system_base for abs_dir in system_dirs: abs_dir = str(abs_dir) @@ -468,7 +482,8 @@ def persist_build_dirs( f"not produce that path." ) - # Store under __egg_system_dirs__/ so it can be restored to the same location + # Store under / so the Dockerfile can + # restore it to the same location with a single COPY to /. # Strip leading / for the destination path dest_dir = system_dest / abs_dir_clean.lstrip("/") dest_dir.parent.mkdir(parents=True, exist_ok=True) diff --git a/sandbox/entrypoint.py b/sandbox/entrypoint.py index 0926deacf9..d12da38cbf 100644 --- a/sandbox/entrypoint.py +++ b/sandbox/entrypoint.py @@ -855,8 +855,10 @@ def restore_prebuilt_deps( for repo_dir in prebuilt_base.iterdir(): if not repo_dir.is_dir(): continue - # __egg_system_dirs__ contains system-level installs (e.g. /usr/local/go) - # already restored by the Dockerfile; skip it here. + # __egg_system_dirs__ held system-level installs (e.g. /usr/local/go) + # in pre-#2999 images; they now persist to /opt/egg-system-dirs (outside + # this tree) and are restored by the Dockerfile either way. Keep the + # skip so older images don't get system dirs restored into a repo. if repo_dir.name == "__egg_system_dirs__": continue # repo_dir is like /opt/prebuilt-deps/owner--repo diff --git a/scripts/await-egg-deploy.sh b/scripts/await-egg-deploy.sh index 8b7aaede07..3f84cff891 100755 --- a/scripts/await-egg-deploy.sh +++ b/scripts/await-egg-deploy.sh @@ -80,10 +80,12 @@ while :; do fi done if [ "$egg_image_pull_failed" -eq 1 ]; then - echo "ERROR: egg-system pods cannot pull image tag '${TAG}' — it is not in k3s." >&2 - echo " A commit, pull, or rebase since your last build moved EGG_IMAGE_TAG." >&2 - echo " 'make deploy' alone only deploys; run 'make redeploy' to rebuild +" >&2 - echo " re-import + deploy on the current tag." >&2 + echo "ERROR: egg-system pods cannot pull image tag '${TAG}'." >&2 + echo " Usual cause: a commit, pull, or rebase since your last build moved" >&2 + echo " EGG_IMAGE_TAG. 'make deploy' alone only deploys; run 'make redeploy'" >&2 + echo " to rebuild + publish + deploy on the current tag." >&2 + echo " If the pod events show 'server gave HTTP response to HTTPS client'," >&2 + echo " k3s was never pointed at the local registry: run 'make registry-setup'." >&2 exit 1 fi diff --git a/scripts/btrfs-reclaim.sh b/scripts/btrfs-reclaim.sh new file mode 100755 index 0000000000..a02b8b4ca0 --- /dev/null +++ b/scripts/btrfs-reclaim.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# +# btrfs-reclaim.sh - Return btrfs over-allocated data chunks to unallocated +# (issue #2999 lever B). +# +# Heavy image churn (docker builds, containerd imports/pulls, tarball +# create/delete) fragments btrfs into many near-empty-but-allocated data +# chunks. statfs counts allocated chunks as used, so kubelet's imagefs +# accounting reads ~86% on a disk that is really ~50% full: image GC fires +# and evicts freshly-published images, DiskPressure sticks and evicts pods — +# and deleting images does NOT help, because chunk allocation only returns +# to the pool via a balance. This script runs that balance. +# +# Usage: btrfs-reclaim.sh [dusage-percent] [mountpoint] +# dusage-percent (default 50): only data chunks <= this %-full are +# compacted. Higher reclaims more but moves more data (slower). +# mountpoint (default /): filesystem to balance. +# +# No-op (exit 0) on non-btrfs filesystems, so callers can invoke it +# unconditionally. Needs sudo (balance is privileged). Safe to run on a live +# system — balance is online — but it is I/O-heavy and can take minutes. +# +set -euo pipefail + +DUSAGE="${1:-50}" +MOUNT="${2:-/}" + +fstype="$(stat -f --format=%T "$MOUNT" 2>/dev/null || echo unknown)" +if [ "$fstype" != "btrfs" ]; then + echo "==> ${MOUNT} is ${fstype}, not btrfs; nothing to reclaim." + exit 0 +fi + +echo "==> btrfs usage on ${MOUNT} before balance:" +sudo btrfs filesystem usage "$MOUNT" | sed -n '1,10p' + +echo "==> Balancing data chunks <=${DUSAGE}% full (online, I/O-heavy, can take minutes)..." +sudo btrfs balance start -dusage="$DUSAGE" "$MOUNT" + +echo "==> btrfs usage on ${MOUNT} after balance:" +sudo btrfs filesystem usage "$MOUNT" | sed -n '1,10p' diff --git a/scripts/check-egg-images-present.sh b/scripts/check-egg-images-present.sh index 2fdb7ba2a9..3d8d6ac1b0 100755 --- a/scripts/check-egg-images-present.sh +++ b/scripts/check-egg-images-present.sh @@ -1,59 +1,122 @@ #!/usr/bin/env bash # # check-egg-images-present.sh - Fail fast, BEFORE `make deploy` mutates the -# cluster, when the egg-*: images are not in k3s's containerd. +# cluster, when the egg images for are not where the cluster +# will look for them. # # EGG_IMAGE_TAG is `git describe --always --dirty`, so it tracks HEAD and # changes on every commit, pull, rebase, or branch checkout. `make redeploy` -# builds, imports, and deploys on one self-consistent tag — but a bare -# `make deploy` after HEAD has moved references a tag whose images were never -# imported. +# builds, publishes, and deploys on one self-consistent tag — but a bare +# `make deploy` after HEAD has moved references a tag that was never +# published. # # await-egg-deploy.sh already detects that, but only AFTER `kubectl apply` has # repointed the live deployments at the missing tag — so a failed bare deploy # leaves the running cluster broken until the operator runs `make redeploy`. -# This pre-flight runs the same authoritative containerd check that k3s-import -# uses (sudo k3s ctr images list), so deploy aborts with the redeploy hint -# without touching the cluster. # -# The check is branch-agnostic: it only asks whether images for the *current* -# tag are present, regardless of how HEAD arrived there. Requires sudo because -# k3s's containerd socket is root-only — same as k3s-import. +# Publishing is split (issue #2999): the registry-subset images (args 3+, +# from EGG_REGISTRY_IMAGES — by default everything except the private-content +# egg-sandbox) are pulled by the cluster from the loopback registry, so the +# registry's HTTP API is their source of truth. The remaining images are +# save+imported, so k3s's containerd is theirs. With no registry (arg 2 +# empty, e.g. CI) every image is checked in containerd — the pre-registry +# behavior, unchanged. # set -euo pipefail -: "${1:?usage: $0 }" +usage="usage: $0 [registry-host:port] [registry-image]..." +: "${1:?$usage}" TAG="$1" +REGISTRY="${2:-}" +shift +[ "$#" -gt 0 ] && shift -# egg-built images rewritten onto the manifests by `make deploy`. Keep in sync -# with the sed rewrites in the deploy target and the k3s-import image list. -IMAGES=(egg-gateway egg-orchestrator egg-sandbox egg-litellm) +# Full egg image set rewritten onto the manifests by `make deploy`. Keep in +# sync with EGG_ALL_IMAGES in the Makefile and reap-stale-egg-images.sh. +ALL_IMAGES=(egg-gateway egg-orchestrator egg-sandbox egg-litellm) -# One listing, reused for every image — k3s ctr is the slow part and needs sudo. -present=$(sudo k3s ctr images list -q) - -missing=() -for img in "${IMAGES[@]}"; do - grep -qx "docker.io/library/$img:$TAG" <<<"$present" || missing+=("$img:$TAG") +# Split ALL_IMAGES into the registry-checked subset and the containerd-checked +# remainder. Without a registry the subset is forced empty. +REGISTRY_IMAGES=() +CONTAINERD_IMAGES=() +for img in "${ALL_IMAGES[@]}"; do + in_registry=0 + if [ -n "$REGISTRY" ]; then + for r in "$@"; do + [ "$r" = "$img" ] && in_registry=1 && break + done + fi + if [ "$in_registry" -eq 1 ]; then + REGISTRY_IMAGES+=("$img") + else + CONTAINERD_IMAGES+=("$img") + fi done -if [ "${#missing[@]}" -gt 0 ]; then - echo "ERROR: egg-system images for tag '${TAG}' are not in k3s: ${missing[*]}" >&2 +missing_registry=() +missing_containerd=() + +if [ "${#REGISTRY_IMAGES[@]}" -gt 0 ]; then + if ! curl -fsS "http://${REGISTRY}/v2/" >/dev/null 2>&1; then + echo "ERROR: local registry at http://${REGISTRY}/v2/ is not answering, so the" >&2 + echo " cluster could not pull egg images even if they were published." >&2 + echo " One-time setup: 'make registry-setup'. If the egg-registry container" >&2 + echo " exists but is stopped: 'docker start egg-registry'." >&2 + exit 1 + fi + + # HEAD against the manifest endpoint: 200 iff the tag exists. The Accept + # header must enumerate the modern manifest types or the registry answers + # 404 for images pushed with current docker/buildkit. + accept='application/vnd.docker.distribution.manifest.v2+json' + accept="${accept}, application/vnd.docker.distribution.manifest.list.v2+json" + accept="${accept}, application/vnd.oci.image.manifest.v1+json" + accept="${accept}, application/vnd.oci.image.index.v1+json" + + for img in "${REGISTRY_IMAGES[@]}"; do + if ! curl -fsS --head -H "Accept: ${accept}" \ + "http://${REGISTRY}/v2/${img}/manifests/${TAG}" >/dev/null 2>&1; then + missing_registry+=("$img:$TAG") + fi + done +fi + +if [ "${#CONTAINERD_IMAGES[@]}" -gt 0 ]; then + # One listing, reused for every image — k3s ctr is the slow part and needs + # sudo because k3s's containerd socket is root-only, same as k3s-import. + present=$(sudo k3s ctr images list -q) + for img in "${CONTAINERD_IMAGES[@]}"; do + grep -qx "docker.io/library/$img:$TAG" <<<"$present" || missing_containerd+=("$img:$TAG") + done +fi + +if [ "${#missing_registry[@]}" -gt 0 ]; then + echo "ERROR: egg images for tag '${TAG}' are not in the local registry (${REGISTRY}): ${missing_registry[*]}" >&2 + echo " HEAD moved (commit/pull/rebase/checkout) since your last build, so" >&2 + echo " 'make deploy' alone references a tag that was never built+pushed." >&2 + echo " Fix: 'make redeploy' rebuilds + publishes + deploys on the current tag." >&2 +fi + +if [ "${#missing_containerd[@]}" -gt 0 ]; then + echo "ERROR: egg-system images for tag '${TAG}' are not in k3s: ${missing_containerd[*]}" >&2 echo " Two known causes:" >&2 echo " 1. HEAD moved (commit/pull/rebase/checkout) since your last build, so" >&2 echo " 'make deploy' alone references a tag that was never built+imported." >&2 - echo " Fix: 'make redeploy' rebuilds + re-imports + deploys on the current tag." >&2 + echo " Fix: 'make redeploy' rebuilds + publishes + deploys on the current tag." >&2 echo " 2. 'make redeploy' DID import these, but kubelet image GC evicted them" >&2 echo " before 'deploy' repointed the pods at the new tag -- they sit" >&2 echo " unreferenced until then, so under disk pressure (root fs over" >&2 echo " imageGCHighThresholdPercent, ~85%) they get collected mid-run." >&2 - echo " Fix: reclaim space in k3s's containerd -- NOT docker, a separate" >&2 - echo " store 'docker system prune' does not touch -- then redeploy, which" >&2 - echo " re-imports everything:" >&2 + echo " Fix: reclaim containerd space -- NOT docker, a separate store" >&2 + echo " 'docker system prune' does not touch -- then redeploy:" >&2 echo " sudo k3s crictl rmi --prune # safe immediately before redeploy" >&2 - echo " 'df -h /' should sit well under 80% before the import. A green deploy" >&2 - echo " now reaps older egg tags automatically (reap-stale-egg-images.sh)." >&2 + echo " On btrfs also run 'make btrfs-reclaim' if 'df -h /' disagrees with" >&2 + echo " reality (issue #2999). A green deploy reaps older egg tags" >&2 + echo " automatically (reap-stale-egg-images.sh)." >&2 +fi + +if [ "${#missing_registry[@]}" -gt 0 ] || [ "${#missing_containerd[@]}" -gt 0 ]; then exit 1 fi -echo "All egg-system images for tag '${TAG}' are present in k3s." +echo "All egg images for tag '${TAG}' are present (registry: ${REGISTRY_IMAGES[*]:-none}; containerd: ${CONTAINERD_IMAGES[*]:-none})." diff --git a/scripts/push-egg-images.sh b/scripts/push-egg-images.sh new file mode 100755 index 0000000000..fea479abcf --- /dev/null +++ b/scripts/push-egg-images.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# +# push-egg-images.sh - Publish the registry-subset egg images to the local +# loopback registry and pre-pull them into k3s's containerd (issue #2999). +# +# This replaces `docker save | k3s ctr images import` for the images in the +# subset. Both legs are layer-aware: `docker push` uploads only the layers +# the registry doesn't have, and the `crictl pull` fetches only the layers +# containerd doesn't have — so a code-only rebuild moves the changed +# tens-of-MB layers instead of re-serializing full images. +# +# The pre-pull is not strictly required for the Deployments (kubelet pulls +# on rollout), but it is load-bearing for two things: +# - images only referenced by later-spawned pods (e.g. egg-sandbox, when +# opted into the registry subset) start instantly instead of pulling on +# first spawn; +# - reap-stale-egg-images.sh's safety gate, which refuses to reap unless +# every just-deployed ref is visible in containerd. +# +# Requires sudo for crictl (containerd's socket is root-only), same as +# k3s-import. +# +set -euo pipefail + +usage="usage: $0 ..." +: "${1:?$usage}" +: "${2:?$usage}" +: "${3:?$usage}" +TAG="$1" +REGISTRY="$2" +shift 2 +# The image list comes from the caller (EGG_REGISTRY_IMAGES in the Makefile); +# by default that EXCLUDES egg-sandbox — it bakes in private repo content and +# must not be pushed to any registry unless the operator opts in there. +IMAGES=("$@") + +# Privacy guard: refuse to push anywhere but the loopback registry. The +# egg images can bake in private repo dependencies (node_modules, .venv, +# anything repositories.yaml build_commands produce), so this flow must never +# publish off-host. The loopback registry is a container on this machine +# bound to 127.0.0.1 — unreachable from the network — which keeps the +# exposure identical to the docker daemon store and k3s's containerd. Anyone +# with a genuine remote-registry use case must build their own push path with +# its own redaction story; this script intentionally has no override. +REG_HOST="${REGISTRY%%:*}" +if [ "$REG_HOST" != "localhost" ] && [ "$REG_HOST" != "127.0.0.1" ]; then + echo "ERROR: refusing to push egg images to non-loopback registry '${REGISTRY}'." >&2 + echo " egg images can contain private repo content; this flow only" >&2 + echo " publishes to a local 127.0.0.1-bound registry (make registry-setup)." >&2 + exit 1 +fi + +if ! curl -fsS "http://${REGISTRY}/v2/" >/dev/null 2>&1; then + echo "ERROR: local registry at http://${REGISTRY}/v2/ is not answering." >&2 + echo " One-time setup: 'make registry-setup' (starts the egg-registry" >&2 + echo " container and points k3s at it). To deploy without a registry," >&2 + echo " set EGG_IMAGE_REGISTRY= (empty) to use the save+import fallback." >&2 + exit 1 +fi + +# Fail fast if k3s was never pointed at the registry: the pre-pull below and +# every later kubelet pull would die with "server gave HTTP response to +# HTTPS client" — after `kubectl apply` has already repointed the cluster. +if [ -d /etc/rancher/k3s ] && ! grep -qs "\"${REGISTRY}\":" /etc/rancher/k3s/registries.yaml; then + echo "ERROR: /etc/rancher/k3s/registries.yaml has no entry for ${REGISTRY}," >&2 + echo " so k3s's containerd cannot pull from the plain-HTTP local registry." >&2 + echo " Run 'make registry-setup' once to write it (restarts the k3s service)." >&2 + exit 1 +fi + +# Push :TAG, then :latest if it differs (the `git describe` fallback outside a +# git checkout sets TAG=latest, in which case the second push would be a +# manifest-only no-op against the same ref — harmless but misleading in logs). +tags=("$TAG") +[ "$TAG" != "latest" ] && tags+=("latest") +for image in "${IMAGES[@]}"; do + for tag in "${tags[@]}"; do + echo ">>> pushing ${REGISTRY}/${image}:${tag}" + # Same digest across the two tags — manifest-only upload, no layer data moves. + docker push "${REGISTRY}/${image}:${tag}" + done +done + +for image in "${IMAGES[@]}"; do + echo ">>> pre-pulling ${REGISTRY}/${image}:${TAG} into k3s containerd" + sudo k3s crictl pull "${REGISTRY}/${image}:${TAG}" +done + +echo "Registry-subset images (${IMAGES[*]}) for tag '${TAG}' published to ${REGISTRY} and pre-pulled into k3s." diff --git a/scripts/reap-stale-egg-images.sh b/scripts/reap-stale-egg-images.sh index 99f7b78f98..0466ff23bb 100755 --- a/scripts/reap-stale-egg-images.sh +++ b/scripts/reap-stale-egg-images.sh @@ -1,22 +1,36 @@ #!/usr/bin/env bash # -# reap-stale-egg-images.sh - After a successful deploy, drop the egg-*: -# images in k3s's containerd that are NOT the just-deployed tag (and NOT the -# floating :latest, which shares content with it). +# reap-stale-egg-images.sh - After a successful deploy, drop the egg images +# that are NOT the just-deployed tag (and NOT the floating :latest, which +# shares content with it). # -# Without this, containerd keeps a full image set -- including the ~12 GB -# egg-sandbox -- for every `git describe` tag ever imported, because +# Without this, containerd keeps a full image set -- including the ~10 GB +# egg-sandbox -- for every `git describe` tag ever deployed, because # `make redeploy` only ever adds the new tag and never removes the old one. # That bloat drives the root filesystem over kubelet's -# imageGCHighThresholdPercent (~85%), and kubelet image GC then evicts the -# freshly-imported egg images mid-`make redeploy`: they are unreferenced until -# `deploy` repoints the pods at the new tag, so they are prime GC fodder. The -# gateway/orchestrator images -- imported first, so already older than -# imageMinimumGCAge (~2 min) by the time the long sandbox/litellm imports -# finish -- are the ones that vanish, while sandbox/litellm (still inside the -# GC-immunity window) survive. check-egg-images-present.sh then aborts the -# deploy. Reaping here bounds containerd so the next redeploy's import spike -# stays under the GC threshold. +# imageGCHighThresholdPercent (~85%), and on btrfs the resulting churn +# over-allocates data chunks into sticky DiskPressure (issue #2999). +# +# Four scopes (issue #2999): +# - containerd (always): remove stale egg refs via crictl rmi. An image's +# authoritative ref form depends on its publish path: registry-subset +# images (args 3+) are authoritative as /:, while +# save+import images (the sandbox, by default) are authoritative as +# docker.io/library/:. Refs in the non-authoritative form for +# their image -- e.g. bare leftovers from before the registry flow -- are +# stale by definition (the digest guard below still protects +# shared-content cases). With no registry every image is bare -- the +# pre-registry behavior, unchanged. +# - the docker daemon store + BuildKit cache (always, lever C): untag stale +# egg tags and cap the build cache, so the build side stops accumulating +# a ~10 GB image per commit ever built. +# - btrfs chunk over-allocation (btrfs hosts, lever B): warn when +# unallocated space runs low, auto-balance when critically low. +# - the registry itself (registry mode only): delete manifests for stale +# tags and run the registry's garbage-collect so blob disk is actually +# reclaimed. Layers are content-addressed, so a redeploy only adds the +# changed layers (~tens of MB) -- but without this reap those still +# accumulate without bound. # # NOTE: this is NOT `crictl rmi --prune`. Prune removes every image no running # container references -- but the egg-sandbox image is referenced only by @@ -30,25 +44,74 @@ # set -euo pipefail -: "${1:?usage: $0 }" +usage="usage: $0 [registry-host:port] [registry-image]..." +: "${1:?$usage}" KEEP_TAG="$1" +REGISTRY="${2:-}" +shift +[ "$#" -gt 0 ] && shift -# The egg image set. This list is the single source of truth WITHIN this script -# (both the safety-gate grep loop below and the awk match-pattern threaded via -# -v image_re are driven from it). Keep in sync ACROSS scripts with -# check-egg-images-present.sh and the k3s-import image list. +# The full egg image set. This list is the single source of truth WITHIN this +# script (the safety-gate loop and the awk match-patterns are driven from it). +# Keep in sync ACROSS scripts with EGG_ALL_IMAGES in the Makefile and +# check-egg-images-present.sh. IMAGES=(egg-gateway egg-orchestrator egg-sandbox egg-litellm) -# Build the awk match alternation from IMAGES so the awk regex below tracks -# additions/removals automatically (e.g. "egg-gateway|egg-orchestrator|..."). -IMAGE_RE="$(IFS='|'; echo "${IMAGES[*]}")" +# Args 3+ name the registry-subset images (EGG_REGISTRY_IMAGES — by default +# everything but the private-content egg-sandbox). An image's AUTHORITATIVE +# containerd ref is /: when it is in the subset and +# docker.io/library/: when it is not (save+import path). With no +# registry the subset is forced empty — every image is bare, the +# pre-registry behavior, unchanged. +REGISTRY_SUBSET=() +if [ -n "$REGISTRY" ]; then + REGISTRY_SUBSET=("$@") +fi +is_registry_image() { + local img="$1" r + for r in "${REGISTRY_SUBSET[@]}"; do + [ "$r" = "$img" ] && return 0 + done + return 1 +} -# Escape regex metacharacters in KEEP_TAG before interpolating into grep -E. +# Escape regex metacharacters before interpolating into grep -E / awk EREs. # Real `git describe` outputs ("v1.2.3-4-gabc123") only contain `.` as a regex -# metacharacter, and the literal-vs-regex false-positive scenario is fictional -# in practice -- but escaping costs nothing and keeps a future tag scheme -# (release candidates, "+" build metadata, etc.) from quietly breaking the gate. -KEEP_TAG_RE="$(printf '%s' "$KEEP_TAG" | sed -e 's/[][\\.*^$+?(){}|/]/\\&/g')" +# metacharacter -- but escaping everything costs nothing and keeps a future +# tag or registry scheme from quietly breaking the gate. `/` is deliberately +# NOT in the class: it is not an ERE metacharacter, and a `\/` inside a +# string-built regex makes gawk warn "escape sequence not a known regexp +# operator" every time these land in the awk programs below. +escape_re() { printf '%s' "$1" | sed -e 's/[][\\.*^$+?(){}|]/\\&/g'; } +KEEP_TAG_RE="$(escape_re "$KEEP_TAG")" + +LEGACY_PREFIX="docker.io/library/" +LEGACY_PREFIX_RE="$(escape_re "$LEGACY_PREFIX")" +REGISTRY_PREFIX_RE="" +[ -n "$REGISTRY" ] && REGISTRY_PREFIX_RE="$(escape_re "${REGISTRY}/")" + +# Image-name alternations split by authority, plus the combined candidate +# pattern. '^$' is the deliberate never-matches placeholder for an empty +# side (refs are never empty strings). +reg_img_alt="" +bare_img_alt="" +for img in "${IMAGES[@]}"; do + if is_registry_image "$img"; then + reg_img_alt="${reg_img_alt:+${reg_img_alt}|}${img}" + else + bare_img_alt="${bare_img_alt:+${bare_img_alt}|}${img}" + fi +done +IMAGE_RE="$(IFS='|'; echo "${IMAGES[*]}")" +if [ -n "$REGISTRY" ]; then + PREFIX_ALT_RE="${REGISTRY_PREFIX_RE}|${LEGACY_PREFIX_RE}" +else + PREFIX_ALT_RE="$LEGACY_PREFIX_RE" +fi +AUTH_REG_RE='^$' +[ -n "$reg_img_alt" ] && AUTH_REG_RE="^${REGISTRY_PREFIX_RE}(${reg_img_alt}):" +AUTH_BARE_RE='^$' +[ -n "$bare_img_alt" ] && AUTH_BARE_RE="^${LEGACY_PREFIX_RE}(${bare_img_alt}):" # `k3s ctr images list` columns: REF TYPE DIGEST SIZE PLATFORMS LABELS. listing="$(sudo k3s ctr images list 2>/dev/null || true)" @@ -63,7 +126,12 @@ listing="$(sudo k3s ctr images list 2>/dev/null || true)" # to run. Skip the reap entirely instead. missing_keep=() for img in "${IMAGES[@]}"; do - if ! grep -qE "^docker\.io/library/${img}:${KEEP_TAG_RE}([[:space:]]|\$)" <<<"$listing"; then + if is_registry_image "$img"; then + expect_prefix_re="$REGISTRY_PREFIX_RE" + else + expect_prefix_re="$LEGACY_PREFIX_RE" + fi + if ! grep -qE "^${expect_prefix_re}${img}:${KEEP_TAG_RE}([[:space:]]|\$)" <<<"$listing"; then missing_keep+=("${img}:${KEEP_TAG}") fi done @@ -72,23 +140,39 @@ if [ "${#missing_keep[@]}" -gt 0 ]; then exit 0 fi -# Reap candidates: every egg-* ref whose tag is neither KEEP_TAG nor latest AND -# whose manifest digest differs from every kept ref's digest. The digest guard -# matters because a commit that does not change an image's build inputs yields a -# tag whose content is byte-identical to the current one -- same digest, same +# Reap candidates: every egg ref (authoritative or legacy prefix) whose tag is +# neither KEEP_TAG nor latest *on the authoritative prefix* AND whose manifest +# digest differs from every kept ref's digest. The digest guard matters +# because a commit that does not change an image's build inputs yields a tag +# whose content is byte-identical to the current one -- same digest, same # image ID. crictl rmi removes by image ID (all of that ID's tags), so removing # such a stale tag by name would take the current image with it -- and the # sandbox image has no running pod to make crictl refuse the removal. Skipping # by digest leaves those harmless duplicate tags in place; they cost no disk. # -# The awk match pattern is built from IMAGE_RE (derived from IMAGES above) and -# threaded in via -v so a fifth image added to IMAGES flows in automatically. -mapfile -t candidates < <(awk -v keep="$KEEP_TAG" -v image_re="$IMAGE_RE" ' - BEGIN { match_re = "^docker\\.io/library/(" image_re "):" } +# The awk match patterns are built from the alternations above so a fifth +# image added to IMAGES flows in automatically. A ref is KEPT when it is the +# authoritative form for its image (registry-qualified for the registry +# subset, bare for the rest) AND carries KEEP_TAG/latest; everything else +# matching an egg image name under either prefix is a candidate. The regex +# fragments ride in via the environment, NOT -v: gawk escape-processes -v +# values, so the `\.` in "docker\.io/library/" would both warn and lose its +# backslash. ENVIRON[] is read verbatim. +mapfile -t candidates < <(KEEP_TAG="$KEEP_TAG" IMAGE_RE="$IMAGE_RE" \ + PREFIX_ALT_RE="$PREFIX_ALT_RE" AUTH_REG_RE="$AUTH_REG_RE" \ + AUTH_BARE_RE="$AUTH_BARE_RE" awk ' + BEGIN { + keep = ENVIRON["KEEP_TAG"] + match_re = "^(" ENVIRON["PREFIX_ALT_RE"] ")(" ENVIRON["IMAGE_RE"] "):" + auth_reg_re = ENVIRON["AUTH_REG_RE"] + auth_bare_re = ENVIRON["AUTH_BARE_RE"] + } $1 ~ match_re { ref = $1; dig = $3 tag = ref; sub(/.*:/, "", tag) - if (tag == keep || tag == "latest") { keepdig[dig] = 1; next } + if ((ref ~ auth_reg_re || ref ~ auth_bare_re) && (tag == keep || tag == "latest")) { + keepdig[dig] = 1; next + } cand_ref[NR] = ref; cand_dig[NR] = dig } END { @@ -98,30 +182,156 @@ mapfile -t candidates < <(awk -v keep="$KEEP_TAG" -v image_re="$IMAGE_RE" ' if [ "${#candidates[@]}" -eq 0 ]; then echo "==> containerd reap: no stale egg images beyond tag '${KEEP_TAG}'/latest." - exit 0 +else + removed=0 + rmi_failed=0 + for ref in "${candidates[@]}"; do + # crictl rmi is best-effort here. The CRI RemoveImage RPC's behavior on an + # in-use image is implementation-defined -- containerd's CRI plugin generally + # allows the removal (the snapshot stays mounted under the running container + # until exit), and there have been "rmi removed image out from under running + # container" reports historically. We do not rely on a refusal: a non-zero + # exit here just means "this ref still exists; we did not remove it," and + # since the digest guard above already excluded any ref that shares an image + # ID with a kept ref, leaving it alone is safe either way. Do NOT relax the + # digest guard or the `|| true` on the Makefile call on the assumption that + # crictl will refuse in-use removals -- it may not. + err="$(sudo k3s crictl rmi "$ref" 2>&1 >/dev/null)" && rc=0 || rc=$? + if [ "$rc" -eq 0 ]; then + echo " reaped $ref" + removed=$((removed + 1)) + else + echo " rmi failed for $ref: ${err:-(no stderr)}" >&2 + rmi_failed=$((rmi_failed + 1)) + fi + done + echo "==> containerd reap: removed ${removed} stale egg image(s); ${rmi_failed} rmi call(s) failed." fi -removed=0 -rmi_failed=0 -for ref in "${candidates[@]}"; do - # crictl rmi is best-effort here. The CRI RemoveImage RPC's behavior on an - # in-use image is implementation-defined -- containerd's CRI plugin generally - # allows the removal (the snapshot stays mounted under the running container - # until exit), and there have been "rmi removed image out from under running - # container" reports historically. We do not rely on a refusal: a non-zero - # exit here just means "this ref still exists; we did not remove it," and - # since the digest guard above already excluded any ref that shares an image - # ID with a kept ref, leaving it alone is safe either way. Do NOT relax the - # digest guard or the `|| true` on the Makefile call on the assumption that - # crictl will refuse in-use removals -- it may not. - err="$(sudo k3s crictl rmi "$ref" 2>&1 >/dev/null)" && rc=0 || rc=$? - if [ "$rc" -eq 0 ]; then - echo " reaped $ref" - removed=$((removed + 1)) - else - echo " rmi failed for $ref: ${err:-(no stderr)}" >&2 - rmi_failed=$((rmi_failed + 1)) +# --- docker daemon store reap (issue #2999 lever C) -------------------------- + +# `make build` mints a fresh : tag set per commit and re-points +# :latest, so without a reap the docker daemon's store (a SEPARATE store from +# containerd's) accumulates one ~10 GB sandbox image per commit ever built. +# Untag every egg ref — bare and registry-qualified — whose tag is neither +# KEEP_TAG nor latest. `docker rmi` without -f only untags while other tags +# reference the same image, so layers shared with the kept tags survive; data +# is freed only when the last referencing tag drops. +if [ -n "$REGISTRY" ]; then + REGISTRY_RE="$(escape_re "$REGISTRY")" + DOCKER_REF_RE="^(${REGISTRY_RE}/)?(${IMAGE_RE}):" +else + DOCKER_REF_RE="^(${IMAGE_RE}):" +fi +# Regex via ENVIRON, not -v, for the same escape-processing reason as the +# containerd awk above. +mapfile -t docker_stale < <(docker images --format '{{.Repository}}:{{.Tag}}' 2>/dev/null | + KEEP_TAG="$KEEP_TAG" DOCKER_REF_RE="$DOCKER_REF_RE" awk ' + $0 ~ ENVIRON["DOCKER_REF_RE"] { + tag = $0; sub(/.*:/, "", tag) + if (tag != ENVIRON["KEEP_TAG"] && tag != "latest" && tag != "") print + }') +if [ "${#docker_stale[@]}" -eq 0 ]; then + echo "==> docker store reap: no stale egg tags beyond '${KEEP_TAG}'/latest." +else + docker_removed=0 + for ref in "${docker_stale[@]}"; do + if docker rmi "$ref" >/dev/null 2>&1; then + docker_removed=$((docker_removed + 1)) + fi + done + echo "==> docker store reap: untagged ${docker_removed}/${#docker_stale[@]} stale egg tag(s)." +fi + +# Cap the BuildKit build cache (the other unbounded docker-side store). The +# default is generous on purpose: the sandbox stage-1 cache (repo deps, +# multi-GB) is expensive to rebuild, and prune is LRU — a too-small cap would +# silently turn every redeploy into a cold dependency build. Override with +# EGG_BUILDKIT_CACHE_CAP. +docker builder prune -f --keep-storage="${EGG_BUILDKIT_CACHE_CAP:-40GB}" >/dev/null 2>&1 || true + +# --- btrfs chunk-reclaim check (issue #2999 lever B) -------------------------- + +# On btrfs, the churn above over-allocates data chunks; statfs counts +# allocated-but-empty chunks as used, kubelet's imagefs accounting crosses its +# ~85% GC threshold on a half-empty disk, and DiskPressure wedges the node. +# Deleting images does NOT return chunks — only a balance does. Auto-balance +# only when unallocated space is critically low (the next redeploy would +# likely wedge); otherwise just point at `make btrfs-reclaim`. +if [ "$(stat -f --format=%T / 2>/dev/null)" = "btrfs" ]; then + unalloc_bytes="$(sudo btrfs filesystem usage -b / 2>/dev/null | + awk '/Device unallocated:/ { print $3 }')" + if [ -n "${unalloc_bytes:-}" ]; then + unalloc_gib=$((unalloc_bytes / 1073741824)) + if [ "$unalloc_gib" -lt 4 ]; then + echo "==> btrfs: only ${unalloc_gib} GiB unallocated — reclaiming chunks now (balance, can take minutes)..." + scripts_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + "${scripts_dir}/btrfs-reclaim.sh" 50 / || echo "==> btrfs reclaim failed (non-fatal)." >&2 + elif [ "$unalloc_gib" -lt 16 ]; then + echo "==> btrfs: ${unalloc_gib} GiB unallocated; run 'make btrfs-reclaim' soon or DiskPressure will wedge redeploys (issue #2999)." + fi fi +fi + +# --- registry-side reap (registry mode only) -------------------------------- + +[ -n "$REGISTRY" ] && [ "${#REGISTRY_SUBSET[@]}" -gt 0 ] || exit 0 + +API="http://${REGISTRY}/v2" +if ! curl -fsS "${API}/" >/dev/null 2>&1; then + echo "==> registry reap: registry at ${API} not answering; skipping." + exit 0 +fi + +# The Accept header must enumerate the modern manifest types or the registry +# answers 404 for images pushed by current docker/buildkit. +ACCEPT='application/vnd.docker.distribution.manifest.v2+json' +ACCEPT="${ACCEPT}, application/vnd.docker.distribution.manifest.list.v2+json" +ACCEPT="${ACCEPT}, application/vnd.oci.image.manifest.v1+json" +ACCEPT="${ACCEPT}, application/vnd.oci.image.index.v1+json" + +# Docker-Content-Digest response header for :, empty if absent. +# The trailing `|| true` is load-bearing: a missing tag makes curl -f exit 22, +# and under set -e + pipefail a failing $(manifest_digest ...) inside an +# assignment would abort the whole reap mid-flight. +manifest_digest() { + curl -fsSI -H "Accept: ${ACCEPT}" "${API}/$1/manifests/$2" 2>/dev/null | + awk 'tolower($1) == "docker-content-digest:" { gsub("\r", "", $2); print $2 }' || true +} + +reg_removed=0 +for img in "${REGISTRY_SUBSET[@]}"; do + tags_json="$(curl -fsS "${API}/${img}/tags/list" 2>/dev/null)" || continue + mapfile -t tags < <(printf '%s' "$tags_json" | + python3 -c 'import json, sys +for t in json.load(sys.stdin).get("tags") or []: + print(t)' 2>/dev/null || true) + + # Digests behind the kept tags. Deleting a manifest by digest unlinks EVERY + # tag that points at it, so any digest shared with a kept tag must survive. + keep_digs=" $(manifest_digest "$img" "$KEEP_TAG") $(manifest_digest "$img" latest) " + + for tag in "${tags[@]}"; do + [ "$tag" = "$KEEP_TAG" ] && continue + [ "$tag" = "latest" ] && continue + dig="$(manifest_digest "$img" "$tag")" + [ -n "$dig" ] || continue + case "$keep_digs" in *" $dig "*) continue ;; esac + if curl -fsS -X DELETE "${API}/${img}/manifests/${dig}" >/dev/null 2>&1; then + echo " registry: deleted ${img}:${tag} (${dig})" + reg_removed=$((reg_removed + 1)) + fi + done done -echo "==> containerd reap: removed ${removed} stale egg image(s); ${rmi_failed} rmi call(s) failed." +# Manifest DELETE only unlinks; the registry's offline GC is what returns +# blob disk. --delete-untagged also collects manifests orphaned by tag +# overwrites (every redeploy re-points :latest, stranding the previous +# manifest untagged-but-stored). Safe here because this runs post-deploy +# when no push is in flight -- do NOT run it concurrently with a build. +if docker exec egg-registry registry garbage-collect --delete-untagged \ + /etc/docker/registry/config.yml >/dev/null 2>&1; then + echo "==> registry reap: deleted ${reg_removed} stale tag manifest(s); garbage-collect done." +else + echo "==> registry reap: deleted ${reg_removed} stale tag manifest(s); garbage-collect FAILED (non-fatal)." >&2 +fi diff --git a/scripts/setup-local-registry.sh b/scripts/setup-local-registry.sh new file mode 100755 index 0000000000..9d52450941 --- /dev/null +++ b/scripts/setup-local-registry.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash +# +# setup-local-registry.sh - One-time host setup for the local image registry +# the fast `make redeploy` flow publishes through (issue #2999). +# +# Why a registry at all: `docker save | k3s ctr images import` always +# re-serializes the FULL image — ~10 GB for egg-sandbox — even when one +# 70 MB source layer changed, and that churn is what fragments btrfs into +# DiskPressure on the local dev host. `docker push` + containerd pull are +# both layer-aware: only changed layers ever move. This script provides the +# two pieces that flow needs: +# +# 1. A `registry:2` container (name: egg-registry) on 127.0.0.1:, +# restart=always, blobs in the `egg-registry-data` docker volume, +# DELETE enabled so reap-stale-egg-images.sh can prune old tags. +# 2. /etc/rancher/k3s/registries.yaml telling k3s's containerd to reach +# the registry over plain HTTP (containerd defaults to HTTPS even for +# localhost). k3s only reads this file at startup, so writing it +# requires one `systemctl restart k3s` — that restarts the k3s +# service, not the running pods. +# +# Idempotent: re-running repairs a stopped container and skips pieces that +# are already in place. The registry binds 127.0.0.1 only — nothing +# off-host can reach it. +# +# Step 2 needs sudo. Run attended (`make registry-setup`) so it can prompt; +# without sudo it prints the exact commands to run and exits non-zero. +# +set -euo pipefail + +REGISTRY="${1:-localhost:5000}" +PORT="${REGISTRY##*:}" +HOST="${REGISTRY%%:*}" +CONTAINER_NAME="egg-registry" +VOLUME_NAME="egg-registry-data" +REGISTRIES_YAML="/etc/rancher/k3s/registries.yaml" + +if [ "$HOST" != "localhost" ] && [ "$HOST" != "127.0.0.1" ]; then + echo "ERROR: this script only sets up a loopback registry (got host '$HOST')." >&2 + echo " A non-local registry needs TLS/auth decisions it can't make for you." >&2 + exit 1 +fi + +# --- 1. Registry container ------------------------------------------------- + +existing="$(docker ps -a --filter "name=^${CONTAINER_NAME}$" --format '{{.Status}}')" +if [ -n "$existing" ]; then + case "$existing" in + Up*) + echo "==> Registry container '${CONTAINER_NAME}' already running." + ;; + *) + echo "==> Starting existing registry container '${CONTAINER_NAME}'..." + docker start "$CONTAINER_NAME" >/dev/null + ;; + esac +else + echo "==> Creating registry container '${CONTAINER_NAME}' on 127.0.0.1:${PORT}..." + # noqa: EGG100 - loopback-only image registry backing the fast redeploy publish path (issue #2999) + docker run -d \ + --name "$CONTAINER_NAME" \ + --restart=always \ + -p "127.0.0.1:${PORT}:5000" \ + -v "${VOLUME_NAME}:/var/lib/registry" \ + -e REGISTRY_STORAGE_DELETE_ENABLED=true \ + registry:2 >/dev/null +fi + +echo "==> Waiting for the registry to answer..." +for _ in $(seq 1 30); do + if curl -fsS "http://${REGISTRY}/v2/" >/dev/null 2>&1; then + break + fi + sleep 1 +done +if ! curl -fsS "http://${REGISTRY}/v2/" >/dev/null 2>&1; then + echo "ERROR: registry container is up but http://${REGISTRY}/v2/ does not answer." >&2 + echo " Check: docker logs ${CONTAINER_NAME}" >&2 + exit 1 +fi +echo " Registry answering at http://${REGISTRY}/v2/" + +# --- 2. k3s registries.yaml ------------------------------------------------ + +if [ ! -d /etc/rancher/k3s ]; then + echo "==> /etc/rancher/k3s not found — k3s is not installed on this host." + echo " Skipping registries.yaml; re-run 'make registry-setup' after 'make k3s-setup'." + exit 0 +fi + +# The mirror entry maps the image-name registry host (what pod specs and +# `docker push` use) to a plain-HTTP endpoint. Without it containerd tries +# HTTPS against the cleartext registry and every pull fails with +# "http: server gave HTTP response to HTTPS client". +wanted_yaml="$( + cat < ${REGISTRIES_YAML} already maps ${REGISTRY}; leaving it alone." + exit 0 + fi + echo "ERROR: ${REGISTRIES_YAML} exists but has no entry for ${REGISTRY}." >&2 + echo " Merge this mirror block into it by hand, then 'sudo systemctl restart k3s':" >&2 + echo "" >&2 + echo "$wanted_yaml" >&2 + exit 1 +fi + +echo "==> Writing ${REGISTRIES_YAML} and restarting k3s to pick it up..." +echo " (k3s only reads registries.yaml at startup; the restart is brief and" +echo " running pod containers reattach when containerd comes back)" +if ! printf '%s\n' "$wanted_yaml" | sudo tee "$REGISTRIES_YAML" >/dev/null; then + echo "ERROR: could not write ${REGISTRIES_YAML} (no sudo?). Run by hand:" >&2 + echo "" >&2 + echo " sudo tee ${REGISTRIES_YAML} <<'EOF'" >&2 + echo "$wanted_yaml" >&2 + echo "EOF" >&2 + echo " sudo systemctl restart k3s" >&2 + exit 1 +fi +sudo chmod 644 "$REGISTRIES_YAML" +sudo systemctl restart k3s + +echo "==> Waiting for the k3s node to come back Ready..." +export KUBECONFIG="${KUBECONFIG:-/etc/rancher/k3s/k3s.yaml}" +for _ in $(seq 1 60); do + if kubectl wait --for=condition=Ready node --all --timeout=5s >/dev/null 2>&1; then + echo "==> Local registry setup complete." + exit 0 + fi + sleep 2 +done +echo "ERROR: k3s did not report Ready within ~2 minutes of the restart." >&2 +echo " Check: systemctl status k3s / journalctl -u k3s" >&2 +exit 1 diff --git a/tests/sandbox/test_docker_setup.py b/tests/sandbox/test_docker_setup.py index f0368e7679..e925c1a30b 100644 --- a/tests/sandbox/test_docker_setup.py +++ b/tests/sandbox/test_docker_setup.py @@ -1005,8 +1005,8 @@ def test_get_build_commands_handles_non_list_persist_system_dirs(self): result = get_build_commands(config) assert result[0]["persist_system_dirs"] == [] - def test_persist_system_dirs_copies_to_prebuilt(self, tmp_path, capsys): - """persist_system_dirs copies absolute-path directories to _system_ subdir.""" + def test_persist_system_dirs_copies_to_system_base(self, tmp_path, capsys): + """persist_system_dirs copies absolute-path directories under system_base.""" from docker_setup import persist_build_dirs # Simulate a system-level Go installation @@ -1016,6 +1016,7 @@ def test_persist_system_dirs_copies_to_prebuilt(self, tmp_path, capsys): (go_dir.parent / "src").mkdir() prebuilt = tmp_path / "prebuilt-deps" + system_base = tmp_path / "egg-system-dirs" repo_deps = tmp_path / "repo-deps" (repo_deps / "org--app").mkdir(parents=True) @@ -1033,10 +1034,11 @@ def test_persist_system_dirs_copies_to_prebuilt(self, tmp_path, capsys): ], repo_deps_base=repo_deps, prebuilt_base=prebuilt, + system_base=system_base, ) - # Should be stored under __egg_system_dirs__/ - dest = prebuilt / "__egg_system_dirs__" / sys_dir.lstrip("/") + # Should be stored under / + dest = system_base / sys_dir.lstrip("/") assert dest.is_dir() assert (dest / "bin" / "go").exists() @@ -1150,6 +1152,7 @@ def test_persist_system_dirs_duplicate_across_repos(self, tmp_path, capsys): (go_dir / "go").write_text("#!/bin/sh\necho go") prebuilt = tmp_path / "prebuilt-deps" + system_base = tmp_path / "egg-system-dirs" repo_deps = tmp_path / "repo-deps" (repo_deps / "org--app1").mkdir(parents=True) (repo_deps / "org--app2").mkdir(parents=True) @@ -1174,10 +1177,11 @@ def test_persist_system_dirs_duplicate_across_repos(self, tmp_path, capsys): ], repo_deps_base=repo_deps, prebuilt_base=prebuilt, + system_base=system_base, ) # Both should succeed (dirs_exist_ok=True merges) - dest = prebuilt / "__egg_system_dirs__" / sys_dir.lstrip("/") + dest = system_base / sys_dir.lstrip("/") assert dest.is_dir() assert (dest / "bin" / "go").exists() @@ -1210,6 +1214,7 @@ def test_persist_system_dirs_overlap_first_writer_wins(self, tmp_path): (repo_deps / "org--app-a").mkdir(parents=True) (repo_deps / "org--app-b").mkdir(parents=True) prebuilt = tmp_path / "prebuilt" + system_base = tmp_path / "egg-system-dirs" # First persist call writes repo-A's content. persist_build_dirs( @@ -1223,6 +1228,7 @@ def test_persist_system_dirs_overlap_first_writer_wins(self, tmp_path): ], repo_deps_base=repo_deps, prebuilt_base=prebuilt, + system_base=system_base, ) # Now mutate the source so repo-B "would" install a different version @@ -1242,9 +1248,10 @@ def test_persist_system_dirs_overlap_first_writer_wins(self, tmp_path): ], repo_deps_base=repo_deps, prebuilt_base=prebuilt, + system_base=system_base, ) - dest = prebuilt / "__egg_system_dirs__" / str(bin_dir).lstrip("/") + dest = system_base / str(bin_dir).lstrip("/") # Shared file: first writer wins (repo-A's content survived). assert (dest / "shared").read_text() == "repo-A version" # Both repo-only files coexist (idempotent merge). diff --git a/tests/scripts/test_dockerignore_drift.py b/tests/scripts/test_dockerignore_drift.py new file mode 100644 index 0000000000..afff6d874f --- /dev/null +++ b/tests/scripts/test_dockerignore_drift.py @@ -0,0 +1,71 @@ +"""Drift guard for the two .dockerignore files. + +Per-Dockerfile `.dockerignore` (BuildKit) does NOT merge with the +root `.dockerignore` — when both exist, BuildKit reads only the per-Dockerfile +one for that build. `sandbox/Dockerfile.dockerignore` mirrors the root file +MINUS the single `repo-deps/` exclusion (the sandbox build uniquely needs that +directory in context for `COPY repo-deps/ /tmp/repo-deps/`). + +If a future change adds an exclusion to the root file but forgets the sandbox +override, the sandbox build context silently bloats — undoing the +1.91 GB → 23 MB savings the override exists to preserve. This test mechanically +enforces the KEEP-IN-SYNC contract documented in the comment headers of both +files (see #2999). +""" + +from __future__ import annotations + +from pathlib import Path + +PROJECT_ROOT = Path(__file__).parent.parent.parent +ROOT_DOCKERIGNORE = PROJECT_ROOT / ".dockerignore" +SANDBOX_DOCKERIGNORE = PROJECT_ROOT / "sandbox" / "Dockerfile.dockerignore" + +# The single line that legitimately differs between the two files. The sandbox +# build needs repo-deps/ in its context; every other build context excludes it. +ALLOWED_SANDBOX_DELTA: set[str] = {"repo-deps/"} + + +def _read_patterns(path: Path) -> set[str]: + """Return the set of effective pattern lines (skip blanks and comments).""" + return { + line.strip() + for line in path.read_text().splitlines() + if line.strip() and not line.lstrip().startswith("#") + } + + +def test_sandbox_dockerignore_mirrors_root() -> None: + """sandbox/Dockerfile.dockerignore must equal root .dockerignore minus repo-deps/.""" + root = _read_patterns(ROOT_DOCKERIGNORE) + sandbox = _read_patterns(SANDBOX_DOCKERIGNORE) + + missing_from_sandbox = (root - sandbox) - ALLOWED_SANDBOX_DELTA + extra_in_sandbox = sandbox - root + + assert not missing_from_sandbox, ( + "Drift between root .dockerignore and sandbox/Dockerfile.dockerignore: " + f"these patterns are in the root file but missing from the sandbox " + f"override: {sorted(missing_from_sandbox)}. Add them to " + "sandbox/Dockerfile.dockerignore or update ALLOWED_SANDBOX_DELTA in " + "this test if the divergence is intentional. See the KEEP-IN-SYNC " + "comment in both files." + ) + assert not extra_in_sandbox, ( + "Drift between root .dockerignore and sandbox/Dockerfile.dockerignore: " + f"these patterns are in the sandbox override but not in the root file: " + f"{sorted(extra_in_sandbox)}. The sandbox override should be a strict " + "subset of the root file (minus repo-deps/)." + ) + + assert ALLOWED_SANDBOX_DELTA.issubset(root), ( + f"ALLOWED_SANDBOX_DELTA expects {sorted(ALLOWED_SANDBOX_DELTA)} to be " + "excluded by the root .dockerignore. If the sandbox-only inclusion no " + "longer applies, delete sandbox/Dockerfile.dockerignore entirely " + "instead of leaving it out of sync." + ) + assert not ALLOWED_SANDBOX_DELTA & sandbox, ( + f"sandbox/Dockerfile.dockerignore must NOT exclude " + f"{sorted(ALLOWED_SANDBOX_DELTA)} — the sandbox build needs it in " + "context (`COPY repo-deps/ /tmp/repo-deps/`)." + ) diff --git a/tests/scripts/test_reap_stale_egg_images.py b/tests/scripts/test_reap_stale_egg_images.py index 287f06d16b..1e062779c2 100644 --- a/tests/scripts/test_reap_stale_egg_images.py +++ b/tests/scripts/test_reap_stale_egg_images.py @@ -33,10 +33,13 @@ def _extract_awk_program() -> str: run instead of letting a stale hardcoded copy mask a regression. """ src = SCRIPT.read_text() - # The script invokes awk with two -v flags: `keep="$KEEP_TAG"` and - # `image_re="$IMAGE_RE"` (the latter is built from IMAGES via IFS join). + # The script feeds KEEP_TAG / IMAGE_RE / PREFIX_ALT_RE / AUTH_REG_RE / + # AUTH_BARE_RE to awk via the environment (NOT -v: gawk escape-processes + # -v values, and the `\.` in "docker\.io/library/" would both warn and + # lose its backslash). Anchor on AUTH_BARE_RE so we pick the containerd + # reap block specifically, not the later docker-store awk. m = re.search( - r"awk -v keep=\"\$KEEP_TAG\" -v image_re=\"\$IMAGE_RE\" '(.+?)'\s*<<<", + r"AUTH_BARE_RE=\"\$AUTH_BARE_RE\"\s+awk\s+'(.+?)'\s*<<<", src, re.DOTALL, ) @@ -57,17 +60,57 @@ def _extract_images() -> list[str]: return m.group(1).split() -def _run_awk(listing: str, keep: str) -> list[str]: - """Run the script's awk block against a synthetic listing.""" +def _run_awk( + listing: str, + keep: str, + *, + registry: str = "", + registry_subset: tuple[str, ...] = (), +) -> list[str]: + """Run the script's awk block against a synthetic listing. + + With `registry=""` and `registry_subset=()` this mirrors the no-registry + case: every egg image is on the legacy docker.io/library/ prefix, and + the registry-authority branch is the never-matching '^$' placeholder. + + With `registry` set (e.g. ``"localhost:5000"``) and `registry_subset` + naming the registry-mode images (e.g. ``("egg-gateway", "egg-orchestrator", + "egg-litellm")``), this mirrors hybrid mode: registry-subset images are + authoritative as ``/:`` while bare-subset images + (the sandbox, by default) are authoritative as ``docker.io/library/...``. + The PREFIX_ALT_RE / AUTH_REG_RE / AUTH_BARE_RE construction below mirrors + reap-stale-egg-images.sh:88-114 — keep both halves in sync. + """ awk = shutil.which("awk") assert awk, "awk binary not on PATH" - image_re = "|".join(_extract_images()) + images = _extract_images() + image_re = "|".join(images) + legacy_prefix_re = r"docker\.io/library/" + if registry: + registry_prefix_re = re.escape(f"{registry}/") + prefix_alt_re = f"{registry_prefix_re}|{legacy_prefix_re}" + else: + registry_prefix_re = "" + prefix_alt_re = legacy_prefix_re + reg_img_alt = "|".join(i for i in images if i in registry_subset) + bare_img_alt = "|".join(i for i in images if i not in registry_subset) + auth_reg_re = f"^{registry_prefix_re}({reg_img_alt}):" if reg_img_alt else "^$" + auth_bare_re = f"^{legacy_prefix_re}({bare_img_alt}):" if bare_img_alt else "^$" + env = { + "KEEP_TAG": keep, + "IMAGE_RE": image_re, + "PREFIX_ALT_RE": prefix_alt_re, + "AUTH_REG_RE": auth_reg_re, + "AUTH_BARE_RE": auth_bare_re, + "PATH": "/usr/bin:/bin", + } result = subprocess.run( - [awk, "-v", f"keep={keep}", "-v", f"image_re={image_re}", _extract_awk_program()], + [awk, _extract_awk_program()], input=listing, capture_output=True, text=True, check=True, + env=env, ) return [line for line in result.stdout.splitlines() if line] @@ -175,6 +218,169 @@ def test_ignores_non_egg_refs(self) -> None: assert reaped == [] +class TestReapHybridMode: + """Hybrid-mode reap: registry-subset images are authoritative as + ``/:``; bare-subset images (the sandbox, by default) + are authoritative as ``docker.io/library/:``. A ref in the + NON-authoritative form for its image (e.g. a bare ``docker.io/library/ + egg-gateway:`` left over from a pre-registry deploy, or a registry- + qualified leftover for the sandbox) is a reap candidate by definition; + the digest guard then spares any candidate that shares an image ID with + a kept ref. + + The default config in this PR has ``EGG_REGISTRY_IMAGES = egg-gateway + egg-orchestrator egg-litellm`` — egg-sandbox stays on the save+import + path because it bakes in private repo content. These tests pin that + split's behavior against the awk extractor's match/auth-ref logic so a + regression in any of registry-subset authoritative form, bare-prefix + non-authoritative form on subset images, mixed authority across + IMAGES[], or the digest guard's interaction with non-authoritative + refs that match `match_re` but neither `auth_reg_re` nor `auth_bare_re`, + silently fails the test instead of silently fails the reap (the failure + mode is no-reap = slow disk fill-up, not destructive — exactly the + creeping regression #2999 was about). + """ + + REGISTRY = "localhost:5000" + # Default subset for this PR: sandbox stays on import (bare-authoritative). + SUBSET = ("egg-gateway", "egg-orchestrator", "egg-litellm") + + def _baseline_kept_refs(self, keep: str) -> list[str]: + """The four authoritative kept refs under the default hybrid split.""" + return [ + _row(f"{self.REGISTRY}/egg-gateway:{keep}", "sha256:aaa"), + _row(f"{self.REGISTRY}/egg-orchestrator:{keep}", "sha256:bbb"), + _row(f"docker.io/library/egg-sandbox:{keep}", "sha256:ccc"), + _row(f"{self.REGISTRY}/egg-litellm:{keep}", "sha256:ddd"), + ] + + def test_subset_image_bare_leftover_sharing_digest_is_spared(self) -> None: + """Registry-subset image kept authoritatively; a bare leftover with the + SAME digest is non-authoritative but shares the image ID, so the digest + guard must spare it — `crictl rmi` by name would yank the current image. + """ + listing = "\n".join( + [ + *self._baseline_kept_refs("v2"), + # Bare leftover for a registry-subset image, same digest as kept. + _row("docker.io/library/egg-gateway:v2", "sha256:aaa"), + ] + ) + reaped = _run_awk(listing, keep="v2", registry=self.REGISTRY, registry_subset=self.SUBSET) + assert reaped == [] + + def test_subset_image_bare_stale_with_distinct_digest_is_reaped(self) -> None: + """Registry-subset image kept authoritatively; a bare leftover with a + DIFFERENT digest is non-authoritative AND digest-distinct — exactly the + case the new auth-aware logic exists to catch (pre-registry deploy left + a stale bare ref behind, no longer shared with any kept ref). + """ + listing = "\n".join( + [ + *self._baseline_kept_refs("v2"), + _row("docker.io/library/egg-gateway:v1", "sha256:old"), + ] + ) + reaped = _run_awk(listing, keep="v2", registry=self.REGISTRY, registry_subset=self.SUBSET) + assert reaped == ["docker.io/library/egg-gateway:v1"] + + def test_bare_subset_image_registry_leftover_sharing_digest_is_spared(self) -> None: + """The sandbox is authoritative as bare under the default subset; a + registry-qualified leftover with the SAME digest is non-authoritative + but shares the image ID, so the digest guard must spare it. + """ + listing = "\n".join( + [ + *self._baseline_kept_refs("v2"), + _row(f"{self.REGISTRY}/egg-sandbox:v2", "sha256:ccc"), + ] + ) + reaped = _run_awk(listing, keep="v2", registry=self.REGISTRY, registry_subset=self.SUBSET) + assert reaped == [] + + def test_bare_subset_image_registry_leftover_distinct_digest_is_reaped(self) -> None: + """Sandbox authoritative as bare; a registry-qualified leftover with a + DIFFERENT digest must be reaped — this is the symmetric case to the + registry-subset-bare-stale path. + """ + listing = "\n".join( + [ + *self._baseline_kept_refs("v2"), + _row(f"{self.REGISTRY}/egg-sandbox:v1", "sha256:oldsand"), + ] + ) + reaped = _run_awk(listing, keep="v2", registry=self.REGISTRY, registry_subset=self.SUBSET) + assert reaped == [f"{self.REGISTRY}/egg-sandbox:v1"] + + def test_mixed_stale_refs_across_all_images(self) -> None: + """Combined: prior-deploy KEEP_TAG=v1 leftovers for every image, in + each image's NON-authoritative prefix form. None share digests with + the kept refs, so all should be reaped. + """ + listing = "\n".join( + [ + *self._baseline_kept_refs("v2"), + # Non-authoritative stale refs for registry-subset images + _row("docker.io/library/egg-gateway:v1", "sha256:old-g"), + _row("docker.io/library/egg-orchestrator:v1", "sha256:old-o"), + _row("docker.io/library/egg-litellm:v1", "sha256:old-l"), + # Non-authoritative stale ref for the bare-subset image + _row(f"{self.REGISTRY}/egg-sandbox:v1", "sha256:old-s"), + ] + ) + reaped = _run_awk(listing, keep="v2", registry=self.REGISTRY, registry_subset=self.SUBSET) + assert sorted(reaped) == sorted( + [ + "docker.io/library/egg-gateway:v1", + "docker.io/library/egg-orchestrator:v1", + "docker.io/library/egg-litellm:v1", + f"{self.REGISTRY}/egg-sandbox:v1", + ] + ) + + def test_authoritative_stale_refs_are_reaped(self) -> None: + """Authoritative stale refs (registry-qualified for subset, bare for + sandbox) with distinct digests are the canonical reap path — exercising + it here under hybrid prefixes ensures the auth-aware regex doesn't + accidentally over-protect refs that ARE in the authoritative form but + carry a stale tag. + """ + listing = "\n".join( + [ + *self._baseline_kept_refs("v2"), + _row(f"{self.REGISTRY}/egg-gateway:v1", "sha256:old-g"), + _row(f"{self.REGISTRY}/egg-orchestrator:v1", "sha256:old-o"), + _row("docker.io/library/egg-sandbox:v1", "sha256:old-s"), + _row(f"{self.REGISTRY}/egg-litellm:v1", "sha256:old-l"), + ] + ) + reaped = _run_awk(listing, keep="v2", registry=self.REGISTRY, registry_subset=self.SUBSET) + assert sorted(reaped) == sorted( + [ + f"{self.REGISTRY}/egg-gateway:v1", + f"{self.REGISTRY}/egg-orchestrator:v1", + "docker.io/library/egg-sandbox:v1", + f"{self.REGISTRY}/egg-litellm:v1", + ] + ) + + def test_latest_authoritative_protects_shared_digest(self) -> None: + """:latest on the authoritative prefix protects refs sharing its digest, + same invariant as no-registry mode but now exercised under the hybrid + prefix split. + """ + listing = "\n".join( + [ + *self._baseline_kept_refs("v2"), + _row(f"{self.REGISTRY}/egg-gateway:latest", "sha256:zzz"), + # Old bare ref shares :latest's digest — must be spared. + _row("docker.io/library/egg-gateway:v1", "sha256:zzz"), + ] + ) + reaped = _run_awk(listing, keep="v2", registry=self.REGISTRY, registry_subset=self.SUBSET) + assert reaped == [] + + class TestReapScriptSafetyGuard: """End-to-end test of the four-image safety gate via PATH shimming.