fix(k3s): preserve Kyber disk headroom

config/k3s/kubelet.conf

@@ -3,4 +3,9 @@ kind: KubeletConfiguration
 # Keep limited parallelism for faster cold starts without allowing a full-node
 # restart to saturate containerd, disk I/O, and CRI request deadlines.
 serializeImagePulls: false
-maxParallelImagePulls: 4
+maxParallelImagePulls: 2
+# Make the single-node disk contract explicit. The ext4 root reserve is managed
+# by the Kyber activation script, keeping ordinary usage below the low watermark
+# while kubelet remains the sole owner of image and container garbage collection.
+imageGCHighThresholdPercent: 85
+imageGCLowThresholdPercent: 80

home-manager/services/k3s/activate.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 # Install the generated k3s service and sync kubeconfig for the current user.
-# @diff@ and @systemctl@ are substituted by pkgs.replaceVars.
+# Command placeholders are substituted by pkgs.replaceVars.
 set -euo pipefail
 
 SERVICE_FILE="$1"
@@ -37,6 +37,42 @@ require_sudo() {
   fi
 }
 
+configure_root_ext4_reserve() {
+  local root_source root_fs_type block_count reserved_blocks target_reserved_blocks
+  local target_reserved_percent=1
+
+  root_source="$(@findmnt@ --noheadings --output SOURCE --target /)"
+  root_fs_type="$(@findmnt@ --noheadings --output FSTYPE --target /)"
+
+  if [ "$root_fs_type" != "ext4" ]; then
+    return 0
+  fi
+  if [ ! -b "$root_source" ]; then
+    echo "Warning: ext4 root source is not a block device: $root_source" >&2
+    return 0
+  fi
+
+  require_sudo || return 0
+  # shellcheck disable=SC2016
+  block_count="$(run_sudo @tune2fs@ -l "$root_source" 2>/dev/null | @awk@ -F: '/^Block count:/ { gsub(/[[:space:]]/, "", $2); print $2 }')"
+  # shellcheck disable=SC2016
+  reserved_blocks="$(run_sudo @tune2fs@ -l "$root_source" 2>/dev/null | @awk@ -F: '/^Reserved block count:/ { gsub(/[[:space:]]/, "", $2); print $2 }')"
+  if [ -z "$block_count" ] || [ -z "$reserved_blocks" ]; then
+    echo "Warning: unable to inspect ext4 reserve on $root_source" >&2
+    return 0
+  fi
+
+  target_reserved_blocks=$((block_count * target_reserved_percent / 100))
+  if [ "$reserved_blocks" -eq "$target_reserved_blocks" ]; then
+    return 0
+  fi
+
+  run_sudo @tune2fs@ -m "$target_reserved_percent" "$root_source"
+  echo "Configured $root_source ext4 reserved blocks to ${target_reserved_percent}%"
+}
+
+configure_root_ext4_reserve
+
 if [ -f "$SERVICE_FILE" ] && ! @diff@ -q "$SERVICE_FILE" "$SYSTEM_SERVICE" >/dev/null 2>&1; then
   require_sudo || exit 0
   run_sudo cp "$SERVICE_FILE" "$SYSTEM_SERVICE"

home-manager/services/k3s/default.nix

@@ -9,8 +9,11 @@ let
   inherit (inputs) host;
   homeDir = config.home.homeDirectory;
   setupScript = pkgs.replaceVars ./activate.sh {
+    awk = "${pkgs.gawk}/bin/awk";
     diff = "${pkgs.diffutils}/bin/diff";
+    findmnt = "${pkgs.util-linux}/bin/findmnt";
     systemctl = "${pkgs.systemd}/bin/systemctl";
+    tune2fs = "${pkgs.e2fsprogs}/bin/tune2fs";
   };
   serviceFile = "${homeDir}/.config/k3s/k3s.service";
 in

named-hosts/kyber/README.md

@@ -49,6 +49,42 @@ Once Tailscale is set up:
 kyber  # Fish abbreviation that runs: ssh ubuntu@kyber
 ```
 
+## k3s Disk Headroom
+
+Kyber runs k3s and its embedded containerd on the root ext4 filesystem. The
+host activation keeps ext4 reserved blocks at 1% and limits kubelet to two
+parallel image pulls. On this 916 GiB volume, Ubuntu's default 5% reserve hid
+about 46 GiB from kubelet and left too little usable headroom during overlapping
+application rollouts.
+
+Kubelet owns image, container, and pod-sandbox garbage collection. Do not add a
+separate `crictl` cleanup timer: deleting CRI objects behind kubelet can race
+active pod lifecycle operations and leave container names or cgroups stuck.
+
+The July 2026 incident was a disk-pressure feedback loop, not a slow Temporal
+queue. Root usage crossed kubelet's 85% image-GC threshold during concurrent
+image pulls. Kubelet attempted to reclaim tens of GiB from a much smaller
+logical image cache while containerd and Kine were already I/O-bound. CRI calls
+timed out, stale tasks accumulated, and Temporal workers could not start new
+chat turns. Reducing the ext4 reserve, limiting pull parallelism, and preserving
+free space prevent that loop.
+
+For diagnosis, check filesystem headroom, I/O pressure, kubelet GC messages,
+and CRI health before restarting services:
+
+```bash
+df -h /
+sudo tune2fs -l "$(findmnt -n -o SOURCE /)" | grep -E 'Block count|Reserved block count'
+sudo journalctl -u k3s --since '30 minutes ago' | grep -E 'image garbage collection|DiskPressure|deadline exceeded'
+sudo k3s crictl info
+```
+
+An ordinary `systemctl restart k3s` intentionally preserves running containers
+because the upstream unit uses `KillMode=process`. If containerd itself is
+wedged, use the installed `k3s-killall.sh` once during an attended recovery,
+then start k3s again. The helper preserves cluster data but terminates every
+running workload, so it is not a timer or routine cleanup mechanism.
+
 ## SSH Key Management
 
 ### Automated Setup

spec/k3s_service_activate_spec.sh

@@ -75,5 +75,15 @@ It 'preserves dry-run command handling'
 When run bash -c "grep 'DRY_RUN_CMD' '$SCRIPT'"
 The output should include 'DRY_RUN_CMD'
 End
+
+It 'keeps one percent of the ext4 root volume reserved'
+When run bash -c "grep 'target_reserved_percent=1' '$SCRIPT'"
+The output should include 'target_reserved_percent=1'
+End
+
+It 'resolves the mounted root block device instead of hard-coding it'
+When run bash -c "grep '@findmnt@ --noheadings --output SOURCE --target /' '$SCRIPT'"
+The output should include '@findmnt@ --noheadings --output SOURCE --target /'
+End
 End
 End