Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions deploy/disk-gc/keep-list.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -115,3 +115,22 @@ generation_keep: 3
# Floor in HOURS. Nothing younger is reaped however superseded. Hours, not days,
# is the whole point — see defect 2 above.
generation_min_age_hours: 6
# Tag shapes that count as a per-build artifact. Every one of these was read off
# .201 on 2026-09-18, not invented. Anything NOT matching is treated as a
# meaningful pointer and is never reaped by the generation rule -- `dev`, `main`,
# a release name, anything in keep_image_tags.
#
# These live here rather than in the script so widening what counts as disposable
# is a config review. Narrow them IN, never out: an unmatched tag means KEEP,
# which is the safe direction, while an over-broad pattern reaps something that
# mattered.
#
# DELIBERATELY ABSENT: the loose `<word>-<hex>` shape (`main-f3d4029`,
# `runtime-fc61f41242dcc01b`). Those are per-sha tags too, but a pattern that
# broad would also match ordinary meaningful tags.
generation_tag_patterns:
- "^\\d{8}T\\d{6}Z?-[0-9a-f]{7,40}$" # 20260918T111918Z-e49dea8f
- "^[0-9a-f]{7,40}-\\d{8}T\\d{6}Z?$" # ffd61901-20260918T111918Z
- "^sha-[0-9a-f]{6,40}$" # sha-ffd6190
- "^[A-Za-z0-9._-]+-sha-[0-9a-f]{6,40}$" # dev-sha-ffd6190
- "^[0-9a-f]{40}$" # bare 40-hex content tag
32 changes: 30 additions & 2 deletions scripts/disk-gc.sh
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,28 @@ SCRATCH="$(mktemp -d "$(dirname "$LOG_FILE")/disk-gc.XXXXXX")"
trap 'rm -rf "$SCRATCH"' EXIT
docker image ls --all --no-trunc --format '{{json .}}' >"$SCRATCH/images.ndjson" 2>/dev/null || : >"$SCRATCH/images.ndjson"
docker ps --all --no-trunc --format '{{json .}}' >"$SCRATCH/ps.ndjson" 2>/dev/null || : >"$SCRATCH/ps.ndjson"
docker ps --all --format '{{.Image}}' 2>/dev/null | sort -u >"$SCRATCH/inuse.txt" || : >"$SCRATCH/inuse.txt"
# OMN-16367: resolve the in-use set by each container's .Image DIGEST.
#
# `docker ps --format '{{.Image}}'` prints the image NAME for a name-referenced
# container and the id only for an id-referenced one. Cross-referencing on that
# field returned a FALSE ZERO on 29 of 30 rows on this host, and a plan built
# from it selected an image a running container was sitting on (a dangling
# postgres layer, 611 MB, measured 2026-09-18 against a live capture).
#
# It matters most exactly where it is least visible: the stability-test, judge
# and lakshman lanes run containers on digests OLDER than their own `:latest`,
# so a name-based check protects the tag while the running layer goes
# unprotected. The digest is the only sound identifier for those.
#
# The name list is still unioned in. It can only ADD protection -- a stale name
# ref keeps an image that might not have needed keeping, which is the safe
# direction -- but it is never the sole source.
{
for cid in $(docker ps -aq 2>/dev/null); do
docker inspect -f '{{.Image}}' "$cid" 2>/dev/null || true
done
docker ps --all --format '{{.Image}}' 2>/dev/null || true
} | sort -u >"$SCRATCH/inuse.txt" || : >"$SCRATCH/inuse.txt"

# Build the stdin JSON envelope from the scratch files (SCRATCH_DIR env tells the
# encoder where to read), then pipe it straight into the planner. Two simple
Expand Down Expand Up @@ -121,8 +142,14 @@ for iid in plan["remove_image_ids"]:

log "Plan: $(echo "$IMAGE_IDS" | grep -c . || true) image(s), $(echo "$CONTAINER_IDS" | grep -c . || true) stopped container(s), builder cache capped at ${BUILDER_CACHE_MAX_SIZE}"

REMOVE_NOMINAL_BYTES="$(echo "$PLAN_JSON" | python3 -c 'import json,sys;print(json.load(sys.stdin).get("remove_nominal_bytes",0))')"
REMOVE_NOMINAL_GB="$(python3 -c "print(f'{${REMOVE_NOMINAL_BYTES}/1000**3:.1f}')")"

if [[ "$EXECUTE" != true ]]; then
log "DRY-RUN — would remove the above. Re-run with --execute to act."
# The number a host-rollout consent row cites. NOMINAL, not reclaim: shared
# base layers collapse, and a measured pass on this host recovered 17.6% of
# nominal. Say nominal, so nobody quotes it as expected disk returned.
log "DRY-RUN — would remove $(echo "$IMAGE_IDS" | grep -c . || true) image(s), ${REMOVE_NOMINAL_GB} GB nominal (NOT reclaim: shared layers collapse). Re-run with --execute to act."
[[ -n "$IMAGE_IDS" ]] && { echo "IMAGES TO REMOVE:"; echo "$IMAGE_IDS"; } >&2
[[ -n "$CONTAINER_IDS" ]] && { echo "STOPPED CONTAINERS TO REMOVE:"; echo "$CONTAINER_IDS"; } >&2
exit 0
Expand Down Expand Up @@ -171,6 +198,7 @@ fi
#
# `--all` stays because including internal/frontend images is correct for a
# scheduled GC, not because it unlocks anything.
log "Plan totals: $(echo "$IMAGE_IDS" | grep -c . || true) image(s), ${REMOVE_NOMINAL_GB} GB nominal (NOT reclaim)"
log "Pruning builder cache to a ceiling of ${BUILDER_CACHE_MAX_SIZE}"
PRUNE_OUT=""
if PRUNE_OUT="$(docker builder prune --all --force --max-used-space "$BUILDER_CACHE_MAX_SIZE" 2>&1)"; then
Expand Down
86 changes: 69 additions & 17 deletions scripts/disk_gc_plan.py
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@
import re
import sys
import urllib.request
from collections.abc import Callable
from collections.abc import Callable, Sequence
from datetime import UTC, datetime
from typing import Any

Expand Down Expand Up @@ -115,32 +115,69 @@
# `runtime-fc61f41242dcc01b`). It is a per-sha tag too, but a pattern that broad
# would also match ordinary meaningful tags, and an unmatched tag simply means
# KEEP -- the safe direction. Narrow it in, never out.
_STAMP_SHA_TAG_RE = re.compile(r"^\d{8}T\d{6}Z?-[0-9a-f]{7,40}$")
_SHA_STAMP_TAG_RE = re.compile(r"^[0-9a-f]{7,40}-\d{8}T\d{6}Z?$")
_PREFIXED_SHA_TAG_RE = re.compile(r"^[A-Za-z0-9._-]+-sha-[0-9a-f]{6,40}$")
_BARE_SHA40_TAG_RE = re.compile(r"^[0-9a-f]{40}$")
# The DEFAULT shapes. They are defaults, not policy: the operative list is
# `generation_tag_patterns` in the keep-list, so widening what counts as a build
# stamp is a config review rather than a code change.
DEFAULT_GENERATION_TAG_PATTERNS: tuple[str, ...] = (
r"^\d{8}T\d{6}Z?-[0-9a-f]{7,40}$", # 20260918T111918Z-e49dea8f
r"^[0-9a-f]{7,40}-\d{8}T\d{6}Z?$", # ffd61901-20260918T111918Z
r"^sha-[0-9a-f]{6,40}$", # sha-ffd6190
r"^[A-Za-z0-9._-]+-sha-[0-9a-f]{6,40}$", # dev-sha-ffd6190
r"^[0-9a-f]{40}$", # bare 40-hex content tag
)

# How many newest generations of a per-sha family to retain, and the floor below
# which nothing is reaped however superseded it is. The floor is in HOURS, not
# days: the deploy agent rebuilds roughly hourly, so a day-scale floor keeps ~72
# which nothing is reaped however superseded. The floor is in HOURS, not days:
# the deploy agent rebuilds roughly hourly, so a day-scale floor keeps ~72
# generations permanently ineligible and a keep-newest-N window never bites.
DEFAULT_GENERATION_KEEP = 3
DEFAULT_GENERATION_MIN_AGE_HOURS = 6

# docker prints SI sizes ("3.66GB", "611MB", "24.73kB").
_SIZE_RE = re.compile(r"^\s*([0-9.]+)\s*([a-zA-Z]*B)\s*$")
_SIZE_UNITS = {
"B": 1,
"KB": 1000,
"MB": 1000**2,
"GB": 1000**3,
"TB": 1000**4,
"KIB": 1024,
"MIB": 1024**2,
"GIB": 1024**3,
"TIB": 1024**4,
}


def _parse_size(value: str) -> int:
"""Docker size string -> bytes. Unparseable -> 0, never an exception.

A wrong total is worse than a missing one only if it is silently wrong, so an
unreadable size contributes nothing rather than guessing.
"""
m = _SIZE_RE.match(value or "")
if not m:
return 0
try:
return int(float(m.group(1)) * _SIZE_UNITS.get(m.group(2).upper(), 0))
except (TypeError, ValueError):
return 0

def _is_build_stamped_tag(tag: str) -> bool:

def _is_build_stamped_tag(tag: str, patterns: Sequence[str]) -> bool:
"""True if the tag is a per-build artifact rather than a meaningful pointer.

Moving pointers (`dev`, `main`), release names and the keep tags all fail this
check and are therefore never reaped by the generation rule.
check and are therefore never reaped by the generation rule. An unparseable
pattern is skipped rather than raising: a broken config must not turn a GC
into a crash, and skipping lands on KEEP.
"""
return bool(
_STAMP_SHA_TAG_RE.match(tag)
or _SHA_STAMP_TAG_RE.match(tag)
or _SHA_TAG_RE.match(tag)
or _PREFIXED_SHA_TAG_RE.match(tag)
or _BARE_SHA40_TAG_RE.match(tag)
)
for pattern in patterns:
try:
if re.match(pattern, tag):
return True
except re.error:
continue
return False


def _is_generation_bounded_repo(repo: str, patterns: list[str]) -> bool:
Expand Down Expand Up @@ -311,6 +348,9 @@ def build_plan(
generation_min_age_hours: float = float(
keep_list.get("generation_min_age_hours", DEFAULT_GENERATION_MIN_AGE_HOURS)
)
generation_tag_patterns: Sequence[str] = (
keep_list.get("generation_tag_patterns") or DEFAULT_GENERATION_TAG_PATTERNS
)

remove_image_ids: list[str] = []
kept_reasons: dict[str, str] = {}
Expand Down Expand Up @@ -390,7 +430,7 @@ def build_plan(
and tag
and tag != "<none>"
and _is_generation_bounded_repo(repo, generation_bounded_repos)
and _is_build_stamped_tag(tag)
and _is_build_stamped_tag(tag, generation_tag_patterns)
):
if age * 24.0 < generation_min_age_hours:
kept_reasons[image_id] = (
Expand Down Expand Up @@ -486,11 +526,23 @@ def build_plan(
image_id: refs_by_id.get(image_id, []) for image_id in remove_image_ids
}

# OMN-16367: summed nominal size of the removal set, so a dry run prints a
# number a host-rollout consent row can cite. NOMINAL, not reclaim: shared
# base layers collapse, and a prior measured pass on this host recovered 17.6%
# of nominal. The caller must not present this as expected disk returned.
size_by_id: dict[str, int] = {}
for image in images:
iid = str(image.get("ID", ""))
if iid and iid not in size_by_id:
size_by_id[iid] = _parse_size(str(image.get("Size", "")))
remove_nominal_bytes = sum(size_by_id.get(i, 0) for i in remove_image_ids)

return {
"min_age_days": min_age_days,
"builder_cache_max_size": builder_cache_max_size,
"remove_image_ids": remove_image_ids,
"remove_image_refs": remove_image_refs,
"remove_nominal_bytes": remove_nominal_bytes,
"remove_container_ids": remove_container_ids,
"kept_reasons": kept_reasons,
}
Expand Down
Loading
Loading