From 2717a48e76395f6d8066825531e235712ff56d96 Mon Sep 17 00:00:00 2001 From: Shun Kakinoki Date: Mon, 20 Jul 2026 03:47:09 +0800 Subject: [PATCH] feat(k3s): harden kyber storage lifecycle --- config/k3s/containerd.mount | 2 +- config/k3s/default.nix | 49 ++++++++ config/k3s/journald.conf | 7 ++ config/k3s/kubelet.conf | 28 +++-- config/k3s/kyber-host-alert.sh | 9 ++ config/k3s/kyber-host-health.service | 7 ++ config/k3s/kyber-host-health.sh | 121 +++++++++++++++++++ config/k3s/kyber-host-health.timer | 12 ++ config/k3s/kyber-smartd.conf | 4 + config/k3s/kyber-smartd.service | 14 +++ home-manager/services/k3s/activate.sh | 68 +++++++++-- home-manager/services/k3s/default.nix | 11 +- named-hosts/kyber/README.md | 65 ++++++++-- named-hosts/kyber/prepare-containerd-disk.sh | 48 +++++--- spec/coverage_spec.sh | 12 +- spec/k3s_service_activate_spec.sh | 82 +++++++++++-- 16 files changed, 479 insertions(+), 60 deletions(-) create mode 100644 config/k3s/journald.conf create mode 100644 config/k3s/kyber-host-alert.sh create mode 100644 config/k3s/kyber-host-health.service create mode 100644 config/k3s/kyber-host-health.sh create mode 100644 config/k3s/kyber-host-health.timer create mode 100644 config/k3s/kyber-smartd.conf create mode 100644 config/k3s/kyber-smartd.service diff --git a/config/k3s/containerd.mount b/config/k3s/containerd.mount index 5af928adc..ce75cf61c 100644 --- a/config/k3s/containerd.mount +++ b/config/k3s/containerd.mount @@ -3,7 +3,7 @@ Description=Dedicated containerd filesystem for K3s Before=k3s.service [Mount] -What=/dev/disk/by-label/k3s-containerd +What=/dev/disk/by-uuid/90f29a7b-38ff-460b-b534-92a02f1412ec Where=/var/lib/rancher/k3s/agent/containerd Type=ext4 Options=noatime diff --git a/config/k3s/default.nix b/config/k3s/default.nix index 32e11b1a2..96716a1ad 100644 --- a/config/k3s/default.nix +++ b/config/k3s/default.nix @@ -14,6 +14,35 @@ let serverActivateScript = pkgs.replaceVars ./activate.sh { inherit galacticaAuthorizedKey; }; + alertScript = pkgs.writeShellApplication { + name = "kyber-host-alert"; + runtimeInputs = [ pkgs.util-linux ]; + text = builtins.readFile ./kyber-host-alert.sh; + }; + healthCheckScript = pkgs.writeShellApplication { + name = "kyber-host-health"; + runtimeInputs = [ + alertScript + pkgs.coreutils + pkgs.gawk + pkgs.gnugrep + pkgs.k3s + pkgs.procps + pkgs.systemd + pkgs.util-linux + ]; + text = builtins.readFile ./kyber-host-health.sh; + }; + healthCheckService = pkgs.replaceVars ./kyber-host-health.service { + inherit healthCheckScript; + }; + smartdConfig = pkgs.replaceVars ./kyber-smartd.conf { + inherit alertScript; + }; + smartdService = pkgs.replaceVars ./kyber-smartd.service { + smartd = "${pkgs.smartmontools}/bin/smartd"; + inherit smartdConfig; + }; in { home.file.".config/k3s/config.yaml" = lib.mkIf isKyber { @@ -38,6 +67,26 @@ in force = true; }; + home.file.".config/k3s/journald.conf.d/10-kyber-limits.conf" = lib.mkIf isKyber { + source = ./journald.conf; + force = true; + }; + + home.file.".config/k3s/kyber-host-health.service" = lib.mkIf isKyber { + source = healthCheckService; + force = true; + }; + + home.file.".config/k3s/kyber-host-health.timer" = lib.mkIf isKyber { + source = ./kyber-host-health.timer; + force = true; + }; + + home.file.".config/k3s/kyber-smartd.service" = lib.mkIf isKyber { + source = smartdService; + force = true; + }; + home.sessionVariables = lib.mkIf (isKyber || isGalactica) { KUBECONFIG = kubeconfig; }; diff --git a/config/k3s/journald.conf b/config/k3s/journald.conf new file mode 100644 index 000000000..7ea2c5186 --- /dev/null +++ b/config/k3s/journald.conf @@ -0,0 +1,7 @@ +[Journal] +Storage=persistent +SystemMaxUse=2G +SystemKeepFree=10G +RuntimeMaxUse=256M +MaxRetentionSec=7day +MaxFileSec=1day diff --git a/config/k3s/kubelet.conf b/config/k3s/kubelet.conf index 97e5275d4..39439c61d 100644 --- a/config/k3s/kubelet.conf +++ b/config/k3s/kubelet.conf @@ -1,12 +1,20 @@ apiVersion: kubelet.config.k8s.io/v1beta1 kind: KubeletConfiguration -# Keep limited parallelism for faster cold starts without allowing a full-node -# restart to saturate containerd, disk I/O, and CRI request deadlines. -serializeImagePulls: false -maxParallelImagePulls: 2 -# Make the single-node disk contract explicit. The ext4 root reserve is managed -# by the Kyber activation script, keeping ordinary usage below the low watermark -# while kubelet remains the sole owner of image and container garbage collection. -# These are kubelet's defaults, pinned here so the host headroom contract is visible. -imageGCHighThresholdPercent: 85 -imageGCLowThresholdPercent: 80 +# Serialize image pulls on the single Kyber containerd SSD. A broad restart must +# not fan out concurrent downloads, unpacking, and snapshot writes. +serializeImagePulls: true +# Start image garbage collection early and preserve enough space for image +# unpacking before kubelet reaches an eviction threshold. +imageGCHighThresholdPercent: 70 +imageGCLowThresholdPercent: 60 +# Keep at least twenty percent free on both the root/control-plane filesystem +# and the dedicated image filesystem. Kubelet remains the sole CRI collector. +evictionHard: + memory.available: "500Mi" + nodefs.available: "20%" + imagefs.available: "20%" + nodefs.inodesFree: "10%" + imagefs.inodesFree: "10%" +# Bound the CRI-managed container log files using kubelet's native rotation. +containerLogMaxSize: 10Mi +containerLogMaxFiles: 3 diff --git a/config/k3s/kyber-host-alert.sh b/config/k3s/kyber-host-alert.sh new file mode 100644 index 000000000..52d203b18 --- /dev/null +++ b/config/k3s/kyber-host-alert.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail + +source_name="${1:-${SMARTD_DEVICE:-kyber-host-health}}" +message="${2:-${SMARTD_MESSAGE:-Kyber host reliability alert}}" +alert="${source_name}: ${message}" + +logger --priority daemon.alert --tag kyber-host-health -- "$alert" +printf 'KYBER ALERT: %s\n' "$alert" | wall --nobanner || true diff --git a/config/k3s/kyber-host-health.service b/config/k3s/kyber-host-health.service new file mode 100644 index 000000000..a7ee37f0f --- /dev/null +++ b/config/k3s/kyber-host-health.service @@ -0,0 +1,7 @@ +[Unit] +Description=Check Kyber storage and CRI reliability +After=var-lib-rancher-k3s-agent-containerd.mount k3s.service + +[Service] +Type=oneshot +ExecStart=@healthCheckScript@/bin/kyber-host-health diff --git a/config/k3s/kyber-host-health.sh b/config/k3s/kyber-host-health.sh new file mode 100644 index 000000000..5edb34522 --- /dev/null +++ b/config/k3s/kyber-host-health.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env bash +set -euo pipefail + +readonly EXPECTED_CONTAINERD_UUID="90f29a7b-38ff-460b-b534-92a02f1412ec" +readonly CONTAINERD_MOUNT="/var/lib/rancher/k3s/agent/containerd" +readonly STATE_DIR="/run/kyber-host-health" +readonly D_STATE_THRESHOLD=3 +readonly D_STATE_SUSTAINED_SAMPLES=5 +readonly IO_SOME_AVG300_THRESHOLD=20 +readonly IO_FULL_AVG300_THRESHOLD=10 +readonly IMAGEFS_USAGE_THRESHOLD=70 +readonly CRI_LATENCY_THRESHOLD_SECONDS=5 +readonly CRI_ERROR_THRESHOLD=5 + +install -d --mode 0755 "$STATE_DIR" + +set_alert() { + local key="$1" + local message="$2" + local marker="$STATE_DIR/${key}.alerted" + + if [ ! -e "$marker" ]; then + kyber-host-alert "$key" "$message" + : >"$marker" + fi +} + +clear_alert() { + local key="$1" + local marker="$STATE_DIR/${key}.alerted" + + if [ -e "$marker" ]; then + logger --priority daemon.notice --tag kyber-host-health -- "$key recovered" + rm -f "$marker" + fi +} + +check_io_pressure() { + local some_avg300 full_avg300 + + # shellcheck disable=SC2016 + some_avg300="$(awk '$1 == "some" { for (i = 1; i <= NF; i++) if ($i ~ /^avg300=/) { sub(/^avg300=/, "", $i); print $i } }' /proc/pressure/io)" + # shellcheck disable=SC2016 + full_avg300="$(awk '$1 == "full" { for (i = 1; i <= NF; i++) if ($i ~ /^avg300=/) { sub(/^avg300=/, "", $i); print $i } }' /proc/pressure/io)" + + if awk -v some="$some_avg300" -v full="$full_avg300" -v some_limit="$IO_SOME_AVG300_THRESHOLD" -v full_limit="$IO_FULL_AVG300_THRESHOLD" 'BEGIN { exit !(some >= some_limit || full >= full_limit) }'; then + set_alert "io-pressure" "sustained I/O PSI is elevated (some avg300=${some_avg300}, full avg300=${full_avg300})" + else + clear_alert "io-pressure" + fi +} + +check_d_state() { + local count_file="$STATE_DIR/d-state.samples" + local d_state_count previous_samples=0 samples=0 + + d_state_count="$(ps --no-headers -eo stat= | awk '$1 ~ /^D/ { count++ } END { print count + 0 }')" + if [ -r "$count_file" ]; then + read -r previous_samples <"$count_file" || previous_samples=0 + fi + + if [ "$d_state_count" -ge "$D_STATE_THRESHOLD" ]; then + samples=$((previous_samples + 1)) + fi + printf '%s\n' "$samples" >"$count_file" + + if [ "$samples" -ge "$D_STATE_SUSTAINED_SAMPLES" ]; then + set_alert "d-state" "${d_state_count} processes have remained in uninterruptible sleep for ${samples} consecutive samples" + elif [ "$samples" -eq 0 ]; then + clear_alert "d-state" + fi +} + +check_image_filesystem() { + local mounted_source mounted_uuid usage_percent + + if ! findmnt --mountpoint "$CONTAINERD_MOUNT" >/dev/null 2>&1; then + set_alert "image-filesystem" "$CONTAINERD_MOUNT is not mounted" + return + fi + + mounted_source="$(findmnt --noheadings --output SOURCE --target "$CONTAINERD_MOUNT")" + mounted_uuid="$(blkid --match-tag UUID --output value "$mounted_source")" + if [ "$mounted_uuid" != "$EXPECTED_CONTAINERD_UUID" ]; then + set_alert "image-filesystem" "$CONTAINERD_MOUNT has UUID $mounted_uuid, expected $EXPECTED_CONTAINERD_UUID" + return + fi + + usage_percent="$(df --output=pcent "$CONTAINERD_MOUNT" | tail -n 1 | tr -cd '0-9')" + if [ "$usage_percent" -ge "$IMAGEFS_USAGE_THRESHOLD" ]; then + set_alert "image-filesystem" "containerd image filesystem usage is ${usage_percent}% (threshold ${IMAGEFS_USAGE_THRESHOLD}%)" + else + clear_alert "image-filesystem" + fi +} + +check_cri() { + local started_at finished_at latency_seconds error_count + + started_at="$(date +%s)" + if ! timeout 15 k3s crictl info >/dev/null 2>&1; then + set_alert "cri-health" "k3s crictl info failed or exceeded 15 seconds" + return + fi + finished_at="$(date +%s)" + latency_seconds=$((finished_at - started_at)) + + error_count="$(journalctl --unit k3s --since '5 minutes ago' --no-pager --quiet 2>/dev/null | + grep -Eci 'DeadlineExceeded|deadline exceeded|FailedPrecondition|failed precondition|reserved (container )?name|failed to (create|stop|remove).*(sandbox|container)|cgroup.*(busy|failed)' || true)" + + if [ "$latency_seconds" -ge "$CRI_LATENCY_THRESHOLD_SECONDS" ] || [ "$error_count" -ge "$CRI_ERROR_THRESHOLD" ]; then + set_alert "cri-health" "CRI latency was ${latency_seconds}s with ${error_count} lifecycle errors in the last five minutes" + else + clear_alert "cri-health" + fi +} + +check_io_pressure +check_d_state +check_image_filesystem +check_cri diff --git a/config/k3s/kyber-host-health.timer b/config/k3s/kyber-host-health.timer new file mode 100644 index 000000000..4cda7fbff --- /dev/null +++ b/config/k3s/kyber-host-health.timer @@ -0,0 +1,12 @@ +[Unit] +Description=Run Kyber storage and CRI reliability checks + +[Timer] +OnBootSec=5min +OnUnitActiveSec=1min +RandomizedDelaySec=15s +Persistent=true +Unit=kyber-host-health.service + +[Install] +WantedBy=timers.target diff --git a/config/k3s/kyber-smartd.conf b/config/k3s/kyber-smartd.conf new file mode 100644 index 000000000..56c7cdef3 --- /dev/null +++ b/config/k3s/kyber-smartd.conf @@ -0,0 +1,4 @@ +# Monitor all SMART-capable host disks. The filesystem UUID pin remains the +# authoritative identity for the containerd mount; smartd monitors physical +# devices because SMART data belongs to the whole SSD, not its partition. +DEVICESCAN -a -n standby,q -s (S/../.././02|L/../01/./03) -W 4,50,60 -m -M exec @alertScript@/bin/kyber-host-alert diff --git a/config/k3s/kyber-smartd.service b/config/k3s/kyber-smartd.service new file mode 100644 index 000000000..2c501d2da --- /dev/null +++ b/config/k3s/kyber-smartd.service @@ -0,0 +1,14 @@ +[Unit] +Description=SMART monitoring for Kyber disks +Documentation=man:smartd(8) man:smartd.conf(5) +Wants=var-lib-rancher-k3s-agent-containerd.mount +After=var-lib-rancher-k3s-agent-containerd.mount + +[Service] +Type=simple +ExecStart=@smartd@ --no-fork --quit=never --configfile=@smartdConfig@ +Restart=on-failure +RestartSec=30s + +[Install] +WantedBy=multi-user.target diff --git a/home-manager/services/k3s/activate.sh b/home-manager/services/k3s/activate.sh index 1ee8ebfec..82d03cdf6 100755 --- a/home-manager/services/k3s/activate.sh +++ b/home-manager/services/k3s/activate.sh @@ -6,9 +6,18 @@ set -euo pipefail SERVICE_FILE="$1" KUBE_DIR="$2" MOUNT_FILE="$3" +JOURNALD_FILE="$4" +HEALTH_SERVICE_FILE="$5" +HEALTH_TIMER_FILE="$6" +SMARTD_SERVICE_FILE="$7" SYSTEM_SERVICE="/etc/systemd/system/k3s.service" SYSTEM_MOUNT="/etc/systemd/system/var-lib-rancher-k3s-agent-containerd.mount" +SYSTEM_JOURNALD="/etc/systemd/journald.conf.d/10-kyber-limits.conf" +SYSTEM_HEALTH_SERVICE="/etc/systemd/system/kyber-host-health.service" +SYSTEM_HEALTH_TIMER="/etc/systemd/system/kyber-host-health.timer" +SYSTEM_SMARTD_SERVICE="/etc/systemd/system/kyber-smartd.service" MOUNT_POINT="/var/lib/rancher/k3s/agent/containerd" +EXPECTED_CONTAINERD_UUID="90f29a7b-38ff-460b-b534-92a02f1412ec" K3S_KUBECONFIG="/etc/rancher/k3s/k3s.yaml" sudo_cmd=() @@ -41,6 +50,20 @@ require_sudo() { return 0 } +sync_root_file() { + local source="$1" + local target="$2" + + if [ ! -f "$source" ] || @diff@ -q "$source" "$target" >/dev/null 2>&1; then + return 1 + fi + + require_sudo || return 1 + run_sudo mkdir -p "$(dirname "$target")" + run_sudo cp -f "$source" "$target" + return 0 +} + configure_root_ext4_reserve() { local root_source root_fs_type filesystem_info block_count reserved_blocks target_reserved_blocks local target_reserved_percent=1 @@ -84,6 +107,16 @@ configure_root_ext4_reserve() { configure_root_ext4_reserve +if @findmnt@ --mountpoint "$MOUNT_POINT" >/dev/null 2>&1; then + mounted_source="$(@findmnt@ --noheadings --output SOURCE --target "$MOUNT_POINT")" + mounted_uuid="$(@blkid@ --match-tag UUID --output value "$mounted_source")" + if [ "$mounted_uuid" != "$EXPECTED_CONTAINERD_UUID" ]; then + echo "Refusing to run k3s with unexpected containerd filesystem UUID: $mounted_uuid" >&2 + echo "Expected $EXPECTED_CONTAINERD_UUID at $MOUNT_POINT" >&2 + exit 1 + fi +fi + if [ -f "$MOUNT_FILE" ] && ! @findmnt@ --mountpoint "$MOUNT_POINT" >/dev/null 2>&1; then if @systemctl@ is-active --quiet k3s; then echo "Refusing to mount the containerd SSD while k3s is running" >&2 @@ -105,22 +138,27 @@ if [ -f "$MOUNT_FILE" ] && ! @findmnt@ --mountpoint "$MOUNT_POINT" >/dev/null 2> fi systemd_changed=0 -if [ -f "$MOUNT_FILE" ] && ! @diff@ -q "$MOUNT_FILE" "$SYSTEM_MOUNT" >/dev/null 2>&1; then - require_sudo || exit 0 - run_sudo cp -f "$MOUNT_FILE" "$SYSTEM_MOUNT" - systemd_changed=1 -fi - -if [ -f "$SERVICE_FILE" ] && ! @diff@ -q "$SERVICE_FILE" "$SYSTEM_SERVICE" >/dev/null 2>&1; then - require_sudo || exit 0 - run_sudo cp -f "$SERVICE_FILE" "$SYSTEM_SERVICE" - systemd_changed=1 -fi +for systemd_file_pair in \ + "$MOUNT_FILE:$SYSTEM_MOUNT" \ + "$SERVICE_FILE:$SYSTEM_SERVICE" \ + "$HEALTH_SERVICE_FILE:$SYSTEM_HEALTH_SERVICE" \ + "$HEALTH_TIMER_FILE:$SYSTEM_HEALTH_TIMER" \ + "$SMARTD_SERVICE_FILE:$SYSTEM_SMARTD_SERVICE"; do + source_file="${systemd_file_pair%%:*}" + target_file="${systemd_file_pair#*:}" + if sync_root_file "$source_file" "$target_file"; then + systemd_changed=1 + fi +done if [ "$systemd_changed" -eq 1 ]; then run_sudo @systemctl@ daemon-reload fi +if sync_root_file "$JOURNALD_FILE" "$SYSTEM_JOURNALD"; then + run_sudo @systemctl@ try-restart systemd-journald.service +fi + if [ -f "$MOUNT_FILE" ]; then require_sudo || exit 0 run_sudo mkdir -p "$MOUNT_POINT" @@ -129,6 +167,14 @@ fi run_sudo @systemctl@ enable --now k3s +if [ -f "$SMARTD_SERVICE_FILE" ]; then + run_sudo @systemctl@ enable --now kyber-smartd.service +fi + +if [ -f "$HEALTH_TIMER_FILE" ]; then + run_sudo @systemctl@ enable --now kyber-host-health.timer +fi + if [ -f "$K3S_KUBECONFIG" ]; then run mkdir -p "$KUBE_DIR" require_sudo || exit 0 diff --git a/home-manager/services/k3s/default.nix b/home-manager/services/k3s/default.nix index 6c82e6f49..615ed22a0 100644 --- a/home-manager/services/k3s/default.nix +++ b/home-manager/services/k3s/default.nix @@ -10,6 +10,7 @@ let homeDir = config.home.homeDirectory; setupScript = pkgs.replaceVars ./activate.sh { awk = "${pkgs.gawk}/bin/awk"; + blkid = "${pkgs.util-linux}/bin/blkid"; diff = "${pkgs.diffutils}/bin/diff"; find = "${pkgs.findutils}/bin/find"; findmnt = "${pkgs.util-linux}/bin/findmnt"; @@ -18,12 +19,20 @@ let }; serviceFile = "${homeDir}/.config/k3s/k3s.service"; mountFile = "${homeDir}/.config/k3s/var-lib-rancher-k3s-agent-containerd.mount"; + journaldFile = "${homeDir}/.config/k3s/journald.conf.d/10-kyber-limits.conf"; + healthServiceFile = "${homeDir}/.config/k3s/kyber-host-health.service"; + healthTimerFile = "${homeDir}/.config/k3s/kyber-host-health.timer"; + smartdServiceFile = "${homeDir}/.config/k3s/kyber-smartd.service"; in lib.mkIf (pkgs.stdenv.isLinux && host.isKyber) { home.activation.setupK3s = lib.hm.dag.entryAfter [ "writeBoundary" ] '' ${pkgs.bash}/bin/bash "${setupScript}" \ "${serviceFile}" \ "${homeDir}/.kube" \ - "${mountFile}" + "${mountFile}" \ + "${journaldFile}" \ + "${healthServiceFile}" \ + "${healthTimerFile}" \ + "${smartdServiceFile}" ''; } diff --git a/named-hosts/kyber/README.md b/named-hosts/kyber/README.md index 2d28a1260..f73c4990a 100644 --- a/named-hosts/kyber/README.md +++ b/named-hosts/kyber/README.md @@ -51,12 +51,14 @@ kyber # Fish abbreviation that runs: ssh ubuntu@kyber ## k3s Containerd SSD -Kyber mounts a dedicated ext4 filesystem labeled `k3s-containerd` at +Kyber mounts the dedicated ext4 filesystem with UUID +`90f29a7b-38ff-460b-b534-92a02f1412ec` at `/var/lib/rancher/k3s/agent/containerd`. The generated `k3s.service` requires that mount, so a missing SSD fails closed instead of silently writing images to -the root filesystem. Linux device letters are not stable across boots; the -systemd mount intentionally resolves the filesystem label rather than -hard-coding `/dev/sda`. +the root filesystem. Linux device letters and user-editable filesystem labels +are not stable identities; the systemd mount intentionally resolves the +verified filesystem UUID rather than hard-coding `/dev/sda`. The +`k3s-containerd` label remains only a human-readable diagnostic aid. On a new or intentionally wiped host, prepare the empty containerd SSD before the first `make switch`: @@ -67,28 +69,36 @@ sudo systemctl stop k3s make switch ``` -The preparation command destroys all data on the selected device. It refuses -to run while k3s is active, while any filesystem on the device is mounted, or -when the existing containerd directory is non-empty. Normal Home Manager -activation never formats disks. +The preparation command destroys all data on the selected device unless it +already contains the expected ext4 UUID. It refuses to run while k3s is active, +while any filesystem on the device is mounted, when the existing containerd +directory is non-empty, or when the pinned UUID resolves to another device. +After formatting, it validates the filesystem type and UUID before mounting it. +Normal Home Manager activation never formats disks. Verify the persistent mount and service dependency after activation: ```bash findmnt /var/lib/rancher/k3s/agent/containerd +findmnt -n -o UUID /var/lib/rancher/k3s/agent/containerd systemctl is-enabled var-lib-rancher-k3s-agent-containerd.mount systemctl show k3s -p Requires -p After sudo systemctl restart k3s findmnt /var/lib/rancher/k3s/agent/containerd ``` -Persistent volumes remain outside the containerd SSD. In particular, an +Persistent volumes remain outside the containerd SSD. The SSD contains only +embedded containerd runtime state: image content, snapshots, metadata, and +temporary runtime data. K3s datastore paths and application PVCs, including +local-path provisioner volumes, remain on the root/storage filesystem. An unrestricted local-path PVC must not be treated as a hard capacity quota. ## k3s Disk Headroom -Host activation keeps the root ext4 reserved blocks at 1% and limits kubelet -to two parallel image pulls. Before containerd received a dedicated SSD, +Host activation keeps the root ext4 reserved blocks at 1%. Kubelet serializes +image pulls, begins image garbage collection at 70% usage, targets 60%, and +evicts before either the root (`nodefs`) or containerd (`imagefs`) filesystem +falls below 20% available space. Before containerd received a dedicated SSD, Ubuntu's default 5% reserve on the 916 GiB root volume hid about 46 GiB from kubelet and left too little usable headroom during overlapping rollouts. @@ -116,6 +126,39 @@ sudo journalctl -u k3s --since '30 minutes ago' | grep -E 'image garbage collect sudo k3s crictl info ``` +## Host Reliability Monitoring and Log Bounds + +The Kyber activation installs native host controls rather than an external CRI +cleaner: + +- journald stores at most 2 GiB persistently and 256 MiB at runtime, retains no + entry longer than seven days, and keeps 10 GiB free; +- kubelet rotates each container log at 10 MiB and retains three files; +- `kyber-smartd.service` uses `smartd` to monitor all SMART-capable physical + disks, including the containerd SSD, and runs short and long self-tests; +- `kyber-host-health.timer` runs a read-only check every minute for five-minute + I/O PSI, five consecutive samples of at least three D-state processes, + containerd image-filesystem usage/identity, CRI probe latency, and recent CRI + lifecycle errors. + +Alerts are deduplicated until recovery. They are written to the journal at +`daemon.alert` priority and broadcast to logged-in sessions with `wall`; a +recovery notice is written when a condition clears. These checks never remove +containers, pod sandboxes, shims, tasks, cgroups, or image content. + +Verify the declarations and inspect current alerts after activation: + +```bash +systemctl status kyber-smartd.service kyber-host-health.timer +systemctl list-timers kyber-host-health.timer +sudo journalctl -t kyber-host-health --since '24 hours ago' +sudo smartctl --scan-open +``` + +Run `sudo smartctl -a` against the physical device reported by +`smartctl --scan-open`; SMART data belongs to the SSD rather than its ext4 +partition. + An ordinary `systemctl restart k3s` intentionally preserves running containers because the upstream unit uses `KillMode=process`. If containerd itself is wedged, use the installed `k3s-killall.sh` once during an attended recovery, diff --git a/named-hosts/kyber/prepare-containerd-disk.sh b/named-hosts/kyber/prepare-containerd-disk.sh index 58e0047f3..9912af149 100755 --- a/named-hosts/kyber/prepare-containerd-disk.sh +++ b/named-hosts/kyber/prepare-containerd-disk.sh @@ -5,6 +5,7 @@ set -euo pipefail DEVICE="${1:-}" CONFIRMATION="${2:-}" MOUNT_LABEL="k3s-containerd" +FILESYSTEM_UUID="90f29a7b-38ff-460b-b534-92a02f1412ec" MOUNT_POINT="/var/lib/rancher/k3s/agent/containerd" if [ -z "$DEVICE" ] || [ "$CONFIRMATION" != "--confirm-wipe" ]; then @@ -34,15 +35,11 @@ if findmnt --mountpoint "$MOUNT_POINT" >/dev/null 2>&1; then exit 1 fi -existing_label_device="" -if [ -e "/dev/disk/by-label/$MOUNT_LABEL" ]; then - existing_label_device="$(readlink -f "/dev/disk/by-label/$MOUNT_LABEL")" - if [ "$existing_label_device" != "$resolved_device" ]; then - echo "Refusing to create a duplicate filesystem label: $MOUNT_LABEL already exists on $existing_label_device" >&2 - exit 1 - fi - if [ "$(lsblk --noheadings --raw --nodeps --output FSTYPE "$DEVICE")" != "ext4" ]; then - echo "Refusing to reuse $DEVICE because its filesystem is not ext4" >&2 +existing_uuid_device="" +if [ -e "/dev/disk/by-uuid/$FILESYSTEM_UUID" ]; then + existing_uuid_device="$(readlink -f "/dev/disk/by-uuid/$FILESYSTEM_UUID")" + if [ "$existing_uuid_device" != "$resolved_device" ]; then + echo "Refusing to create duplicate filesystem UUID $FILESYSTEM_UUID; it already exists on $existing_uuid_device" >&2 exit 1 fi fi @@ -58,15 +55,34 @@ if [ -d "$MOUNT_POINT" ]; then fi fi -if [ -z "$existing_label_device" ]; then - sudo wipefs --all "$DEVICE" - sudo mkfs.ext4 -F -L "$MOUNT_LABEL" "$DEVICE" - sudo udevadm settle +current_uuid="$(sudo blkid --match-tag UUID --output value "$DEVICE" 2>/dev/null || true)" +current_fs_type="$(sudo blkid --match-tag TYPE --output value "$DEVICE" 2>/dev/null || true)" + +if [ "$current_uuid" = "$FILESYSTEM_UUID" ]; then + if [ "$current_fs_type" != "ext4" ]; then + echo "Refusing to reuse $DEVICE because UUID $FILESYSTEM_UUID is not on an ext4 filesystem" >&2 + exit 1 + fi + echo "Reusing $DEVICE with its existing pinned ext4 filesystem UUID" else - echo "Reusing $DEVICE with its existing $MOUNT_LABEL ext4 filesystem" + sudo wipefs --all "$DEVICE" + sudo mkfs.ext4 -F -L "$MOUNT_LABEL" -U "$FILESYSTEM_UUID" "$DEVICE" fi + +sudo udevadm settle +prepared_uuid="$(sudo blkid --match-tag UUID --output value "$DEVICE")" +prepared_fs_type="$(sudo blkid --match-tag TYPE --output value "$DEVICE")" +if [ "$prepared_uuid" != "$FILESYSTEM_UUID" ] || [ "$prepared_fs_type" != "ext4" ]; then + echo "Prepared filesystem identity mismatch on $DEVICE: UUID=$prepared_uuid TYPE=$prepared_fs_type" >&2 + exit 1 +fi +if [ ! -e "/dev/disk/by-uuid/$FILESYSTEM_UUID" ]; then + echo "Pinned filesystem UUID path was not created: /dev/disk/by-uuid/$FILESYSTEM_UUID" >&2 + exit 1 +fi + sudo mkdir -p "$MOUNT_POINT" -sudo mount "/dev/disk/by-label/$MOUNT_LABEL" "$MOUNT_POINT" +sudo mount "/dev/disk/by-uuid/$FILESYSTEM_UUID" "$MOUNT_POINT" mounted_source="$(findmnt --noheadings --output SOURCE --target "$MOUNT_POINT")" resolved_source="$(readlink -f "$mounted_source")" @@ -75,5 +91,5 @@ if [ "$resolved_source" != "$resolved_device" ]; then exit 1 fi -echo "Prepared $DEVICE as $MOUNT_LABEL and mounted it at $MOUNT_POINT" +echo "Prepared $DEVICE as UUID $FILESYSTEM_UUID ($MOUNT_LABEL) and mounted it at $MOUNT_POINT" echo "Run make switch to install the persistent systemd mount dependency" diff --git a/spec/coverage_spec.sh b/spec/coverage_spec.sh index c4c24f83d..599676828 100644 --- a/spec/coverage_spec.sh +++ b/spec/coverage_spec.sh @@ -413,6 +413,8 @@ config/hyprland/scripts/record-screen.sh config/hyprland/scripts/toggle-terminal.sh config/k3s/activate-client.sh config/k3s/activate.sh +config/k3s/kyber-host-alert.sh +config/k3s/kyber-host-health.sh config/noctalia/ac-idle-inhibit.sh config/noctalia/lock-before-sleep.sh config/noctalia/quit-active-app.sh @@ -500,8 +502,14 @@ scripts/update-moshi-hooks.sh scripts/upgrade-overlays.sh scripts/wallpaper-power-check.sh" -# Get actual scripts from git (excluding spec directory) -actual_scripts=$(git ls-files '*.sh' 2>/dev/null | grep -v '^spec/' | sort) +# Include untracked additions so the local coverage check is useful before +# staging or committing a newly added script. +actual_scripts=$( + { + git ls-files '*.sh' 2>/dev/null + git ls-files --others --exclude-standard '*.sh' 2>/dev/null + } | grep -v '^spec/' | sort -u +) expected_scripts=$(echo "$covered_scripts" | sort) When run bash -c "diff <(echo '$actual_scripts') <(echo '$expected_scripts') || echo 'MISMATCH: Update coverage_spec.sh when adding new shell scripts'" diff --git a/spec/k3s_service_activate_spec.sh b/spec/k3s_service_activate_spec.sh index 248e9019d..08680a3ca 100755 --- a/spec/k3s_service_activate_spec.sh +++ b/spec/k3s_service_activate_spec.sh @@ -32,8 +32,8 @@ The output should include 'Requires=var-lib-rancher-k3s-agent-containerd.mount' The status should be success End -It 'mounts the labeled SSD at the containerd path' -When run bash -c "grep -qxF 'What=/dev/disk/by-label/k3s-containerd' '$MOUNT_UNIT' && grep -qxF 'Where=/var/lib/rancher/k3s/agent/containerd' '$MOUNT_UNIT'" +It 'mounts the verified SSD filesystem UUID at the containerd path' +When run bash -c "grep -qxF 'What=/dev/disk/by-uuid/90f29a7b-38ff-460b-b534-92a02f1412ec' '$MOUNT_UNIT' && grep -qxF 'Where=/var/lib/rancher/k3s/agent/containerd' '$MOUNT_UNIT'" The status should be success End @@ -44,7 +44,58 @@ End Describe 'container garbage collection ownership' It 'does not configure external CRI garbage collection' -When run bash -c "! grep -R -q 'crictl\|containerd-cleanup\|pods --state NotReady' config/k3s home-manager/services/k3s" +When run bash -c "! grep -R -Eq 'crictl (rm|rmp)|containerd-cleanup|pods --state NotReady|ctr .* (rm|delete)' config/k3s home-manager/services/k3s" +The status should be success +End +End + +Describe 'kubelet storage policy' +KUBELET_CONFIG="$PWD/config/k3s/kubelet.conf" + +It 'serializes image pulls without a parallel-pull override' +When run bash -c "grep -qxF 'serializeImagePulls: true' '$KUBELET_CONFIG' && ! grep -q '^maxParallelImagePulls:' '$KUBELET_CONFIG'" +The status should be success +End + +It 'collects images at seventy percent and targets sixty percent' +When run bash -c "grep -qxF 'imageGCHighThresholdPercent: 70' '$KUBELET_CONFIG' && grep -qxF 'imageGCLowThresholdPercent: 60' '$KUBELET_CONFIG'" +The status should be success +End + +It 'preserves twenty percent on root and image filesystems' +When run bash -c "grep -qxF ' nodefs.available: \"20%\"' '$KUBELET_CONFIG' && grep -qxF ' imagefs.available: \"20%\"' '$KUBELET_CONFIG'" +The status should be success +End + +It 'uses kubelet native container log rotation' +When run bash -c "grep -qxF 'containerLogMaxSize: 10Mi' '$KUBELET_CONFIG' && grep -qxF 'containerLogMaxFiles: 3' '$KUBELET_CONFIG'" +The status should be success +End +End + +Describe 'host reliability controls' +JOURNALD_CONFIG="$PWD/config/k3s/journald.conf" +HEALTH_CHECK="$PWD/config/k3s/kyber-host-health.sh" +HEALTH_TIMER="$PWD/config/k3s/kyber-host-health.timer" +SMARTD_CONFIG="$PWD/config/k3s/kyber-smartd.conf" + +It 'caps persistent and runtime journal storage with journald settings' +When run bash -c "grep -qxF 'SystemMaxUse=2G' '$JOURNALD_CONFIG' && grep -qxF 'RuntimeMaxUse=256M' '$JOURNALD_CONFIG' && grep -qxF 'MaxRetentionSec=7day' '$JOURNALD_CONFIG'" +The status should be success +End + +It 'monitors SMART-capable disks with native smartd alerts' +When run bash -c "grep -q '^DEVICESCAN -a ' '$SMARTD_CONFIG' && grep -q -- '-M exec @alertScript@/bin/kyber-host-alert' '$SMARTD_CONFIG'" +The status should be success +End + +It 'checks each incident reliability signal without mutating CRI state' +When run bash -c "grep -q '/proc/pressure/io' '$HEALTH_CHECK' && grep -q 'D_STATE_SUSTAINED_SAMPLES=5' '$HEALTH_CHECK' && grep -q 'IMAGEFS_USAGE_THRESHOLD=70' '$HEALTH_CHECK' && grep -q 'k3s crictl info' '$HEALTH_CHECK' && grep -q 'journalctl --unit k3s' '$HEALTH_CHECK' && ! grep -Eq 'crictl (rm|rmp)|ctr .* (rm|delete)|kill .*shim' '$HEALTH_CHECK'" +The status should be success +End + +It 'runs the read-only reliability check every minute' +When run bash -c "grep -qxF 'OnUnitActiveSec=1min' '$HEALTH_TIMER' && grep -qxF 'Persistent=true' '$HEALTH_TIMER'" The status should be success End End @@ -92,6 +143,11 @@ The output should include 'var-lib-rancher-k3s-agent-containerd.mount' The status should be success End +It 'installs native log limits and reliability monitors' +When run bash -c "grep -q '/etc/systemd/journald.conf.d/10-kyber-limits.conf' '$SCRIPT' && grep -q 'enable --now kyber-smartd.service' '$SCRIPT' && grep -q 'enable --now kyber-host-health.timer' '$SCRIPT'" +The status should be success +End + It 'orders the k3s config hook after the mount hook' When run grep 'entryAfter \[ "setupK3s" \]' "$SERVER_MODULE" The output should include 'entryAfter [ "setupK3s" ]' @@ -103,6 +159,11 @@ When run bash -c "grep -q 'Refusing to mount the containerd SSD while k3s is run The status should be success End +It 'rejects an already-mounted filesystem with the wrong UUID' +When run bash -c "grep -q 'EXPECTED_CONTAINERD_UUID=\"90f29a7b-38ff-460b-b534-92a02f1412ec\"' '$SCRIPT' && grep -q 'mounted_uuid=.*@blkid@' '$SCRIPT' && grep -q 'unexpected containerd filesystem UUID' '$SCRIPT'" +The status should be success +End + It 'inspects the root-owned containerd directory through sudo' When run grep 'run_sudo @find@ "$MOUNT_POINT"' "$SCRIPT" The output should include 'run_sudo @find@ "$MOUNT_POINT"' @@ -150,9 +211,9 @@ The output should include 'systemctl is-active --quiet k3s' The status should be success End -It 'creates the stable containerd filesystem label' -When run grep 'mkfs.ext4 -F -L "$MOUNT_LABEL"' "$PREPARE_SCRIPT" -The output should include 'mkfs.ext4 -F -L "$MOUNT_LABEL"' +It 'creates the pinned containerd filesystem UUID' +When run grep 'mkfs.ext4 -F -L "$MOUNT_LABEL" -U "$FILESYSTEM_UUID"' "$PREPARE_SCRIPT" +The output should include 'mkfs.ext4 -F -L "$MOUNT_LABEL" -U "$FILESYSTEM_UUID"' The status should be success End @@ -162,8 +223,13 @@ The output should include 'sudo find "$MOUNT_POINT"' The status should be success End -It 'reuses the label only when it resolves to the selected device' -When run bash -c "grep -q 'existing_label_device=.*readlink -f' '$PREPARE_SCRIPT' && grep -q '\"\$existing_label_device\" != \"\$resolved_device\"' '$PREPARE_SCRIPT'" +It 'rejects a pinned UUID already owned by another device' +When run bash -c "grep -q 'existing_uuid_device=.*readlink -f' '$PREPARE_SCRIPT' && grep -q '\"\$existing_uuid_device\" != \"\$resolved_device\"' '$PREPARE_SCRIPT'" +The status should be success +End + +It 'validates the type and UUID before mounting by UUID' +When run bash -c "grep -q 'prepared_uuid=.*blkid' '$PREPARE_SCRIPT' && grep -q 'prepared_fs_type=.*blkid' '$PREPARE_SCRIPT' && grep -q 'mount \"/dev/disk/by-uuid/\$FILESYSTEM_UUID\"' '$PREPARE_SCRIPT'" The status should be success End End