From e34ea7b3eda5d1359f8eda9e05c8471d73b682c5 Mon Sep 17 00:00:00 2001 From: Tanguille <91473554+Tanguille@users.noreply.github.com> Date: Sun, 2 Aug 2026 15:49:05 +0200 Subject: [PATCH] feat(observability): alert on hermes cron and kopiur backup health Backups ran hourly for 18d with LAST-VERIFIED empty on all 18 policies, and a cron stayed dark 40h, because nothing watched either. The signal already existed on disk and in VictoriaMetrics; only the rules were missing. - hermes-rules: 5 alerts over the cron health metrics pushed from the PVC. Every expr is wrapped in last_over_time(...[30m]) because the series are pushed, not scraped, and VM's 5m staleness makes a bare selector return nothing between pushes -- verified live: the bare selector returns 0 series while the wrapped one returns 14. - kopiur-rules: freshness, metric-absence, empty-repo and leader alerts. Group is kopiur.freshness.rules and the freshness alert is KopiurBackupOverdue: the chart ships its own kopiur.rules group with a KopiurBackupStale on the same metric at the same severity, which would have produced identical alert fingerprints. - SnapshotPolicy: enable weekly quick verification. It is opt-in and nobody had opted in, so a repo that stopped being restorable would have read healthy until a restore was attempted. - flux-operator: bind the anonymous flux-admin group to the chart's read-only flux-web-user instead of flux-web-admin, which granted create on batch/jobs, patch/restart on workloads and delete on pods to anyone reaching the internal gateway. Restore the chart's default networkPolicy. Anonymous read of Secrets remains until the UI gets real auth or the route is dropped. - gitignore: age.key* -- *.key and /age.key both miss suffixed backups like age.key.bak-x25519-only, which decrypts the tracked sops files in this public repo. Rules validated with kubectl apply --dry-run=server and every expression run against live VictoriaMetrics. --- .gitignore | 4 +- .../apps/ai/hermes/app/kustomization.yaml | 1 + .../apps/ai/hermes/app/prometheusrule.yaml | 79 +++++++++++++++++++ .../flux-operator/app/clusterrolebinding.yaml | 7 +- .../flux-operator/app/helmrelease.yaml | 5 +- .../kopiur/app/kustomization.yaml | 1 + .../kopiur/app/prometheusrule.yaml | 73 +++++++++++++++++ .../kopiur/backup/snapshotpolicy.yaml | 10 +++ 8 files changed, 177 insertions(+), 3 deletions(-) create mode 100644 kubernetes/apps/ai/hermes/app/prometheusrule.yaml create mode 100644 kubernetes/apps/kopiur-system/kopiur/app/prometheusrule.yaml diff --git a/.gitignore b/.gitignore index ab9a610dc1..4b67afc78e 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,9 @@ *.pub *.key *.decrypted~*.yaml -/age.key +# `*.key` misses suffixed backups like age.key.bak-x25519-only, which decrypt the +# tracked sops files in this PUBLIC repo. Supersedes the old `/age.key`. +age.key* /cloudflare-tunnel.json bootstrap/github-deploy.key bootstrap/github-deploy.key.pub diff --git a/kubernetes/apps/ai/hermes/app/kustomization.yaml b/kubernetes/apps/ai/hermes/app/kustomization.yaml index 9aac8de8f5..b06016f0be 100644 --- a/kubernetes/apps/ai/hermes/app/kustomization.yaml +++ b/kubernetes/apps/ai/hermes/app/kustomization.yaml @@ -5,4 +5,5 @@ kind: Kustomization resources: - helmrelease.yaml + - prometheusrule.yaml - secret.sops.yaml diff --git a/kubernetes/apps/ai/hermes/app/prometheusrule.yaml b/kubernetes/apps/ai/hermes/app/prometheusrule.yaml new file mode 100644 index 0000000000..83f2ba0be2 --- /dev/null +++ b/kubernetes/apps/ai/hermes/app/prometheusrule.yaml @@ -0,0 +1,79 @@ +--- +# yaml-language-server: $schema=https://k8s-schemas.home-operations.com/monitoring.coreos.com/prometheusrule_v1.json +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: hermes-rules +spec: + groups: + - name: hermes.rules + # scripts/cron_health_export.py (on the hermes PVC, not in git) pushes + # cron/jobs.json state every 10m. Pushed, not scraped, so VM's 5m default + # staleness makes a bare instant selector return nothing between pushes: + # absent() would flap and the failure rules would go blind exactly half the + # time. last_over_time() pins every rule to the newest real sample instead of + # the staleness window. + rules: + # The scheduler already records per-job status in cron/jobs.json; nothing read + # it, so a dead cron stayed dark 40h. + - alert: HermesCronJobFailed + expr: |- + last_over_time(hermes_cron_last_status_ok[30m]) == 0 + for: 15m + annotations: + summary: >- + Hermes cron {{ $labels.name }} last run failed — it will keep failing on + schedule until the cause is fixed + labels: + severity: warning + + # Delivery is a separate failure from execution: the job can succeed and the + # Discord post still vanish, which reads green everywhere else. + - alert: HermesCronDeliveryFailed + expr: |- + last_over_time(hermes_cron_delivery_failed[30m]) == 1 + for: 15m + annotations: + summary: >- + Hermes cron {{ $labels.name }} ran but could not deliver its output — + the result is lost, not late + labels: + severity: warning + + # Already excludes disabled and paused jobs at export time. One hour of slack + # absorbs a long-running predecessor without flapping. + - alert: HermesCronOverdue + expr: |- + last_over_time(hermes_cron_overdue_seconds[30m]) > 3600 + for: 15m + annotations: + summary: >- + Hermes cron {{ $labels.name }} is {{ $value | humanizeDuration }} past its + scheduled run — the scheduler is not firing it + labels: + severity: warning + + # The exporter runs inside hermes, so a stale heartbeat means hermes itself is + # down. Without this the other three rules go quiet exactly when they matter. + - alert: HermesCronExporterStale + expr: |- + time() - last_over_time(hermes_cron_export_timestamp[2h]) > 1800 + for: 5m + annotations: + summary: >- + Hermes cron health metrics are {{ $value | humanizeDuration }} stale — + cron alerting is blind, check the hermes pod + labels: + severity: warning + + # No for:: absent() over a 2h-integrated window cannot flap, and any delay here + # reopens the gap where Stale has already resolved but Absent has not yet fired, + # which reports "recovered" in the middle of an outage. + - alert: HermesCronExporterAbsent + expr: |- + absent(last_over_time(hermes_cron_export_timestamp[2h])) + annotations: + summary: >- + No Hermes cron health metrics at all — the exporter never ran or was removed + labels: + severity: warning diff --git a/kubernetes/apps/flux-system/flux-operator/app/clusterrolebinding.yaml b/kubernetes/apps/flux-system/flux-operator/app/clusterrolebinding.yaml index dbd53a48e1..3f7b49b5f6 100644 --- a/kubernetes/apps/flux-system/flux-operator/app/clusterrolebinding.yaml +++ b/kubernetes/apps/flux-system/flux-operator/app/clusterrolebinding.yaml @@ -10,5 +10,10 @@ subjects: apiGroup: rbac.authorization.k8s.io roleRef: kind: ClusterRole - name: flux-web-admin + # The web UI authenticates as Anonymous, so this role is granted to anyone who can + # reach envoy-internal. flux-web-admin adds create on batch/jobs (arbitrary pod + # execution), patch/restart on workloads and delete on pods; flux-web-user is the + # chart's read-only variant. It still carries get/list/watch on *, which includes + # Secrets -- only real auth or dropping web.httpRoute closes that. + name: flux-web-user apiGroup: rbac.authorization.k8s.io diff --git a/kubernetes/apps/flux-system/flux-operator/app/helmrelease.yaml b/kubernetes/apps/flux-system/flux-operator/app/helmrelease.yaml index aed7da1108..f359795661 100644 --- a/kubernetes/apps/flux-system/flux-operator/app/helmrelease.yaml +++ b/kubernetes/apps/flux-system/flux-operator/app/helmrelease.yaml @@ -27,5 +27,8 @@ spec: parentRefs: - name: envoy-internal namespace: network + # Chart default is true. Caps pod ingress to 9080 (web) and 8080 (metrics, kept + # because serviceMonitor.create is on); `from` is namespaceSelector:{}, so reach + # to the anonymous UI is unchanged — the read-only roleRef is what limits it. networkPolicy: - create: false + create: true diff --git a/kubernetes/apps/kopiur-system/kopiur/app/kustomization.yaml b/kubernetes/apps/kopiur-system/kopiur/app/kustomization.yaml index 660b70723d..8be8c76c4c 100644 --- a/kubernetes/apps/kopiur-system/kopiur/app/kustomization.yaml +++ b/kubernetes/apps/kopiur-system/kopiur/app/kustomization.yaml @@ -5,3 +5,4 @@ kind: Kustomization resources: - helmrelease.yaml - ocirepository.yaml + - prometheusrule.yaml diff --git a/kubernetes/apps/kopiur-system/kopiur/app/prometheusrule.yaml b/kubernetes/apps/kopiur-system/kopiur/app/prometheusrule.yaml new file mode 100644 index 0000000000..b316c5e957 --- /dev/null +++ b/kubernetes/apps/kopiur-system/kopiur/app/prometheusrule.yaml @@ -0,0 +1,73 @@ +--- +# yaml-language-server: $schema=https://k8s-schemas.home-operations.com/monitoring.coreos.com/prometheusrule_v1.json +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: kopiur-rules +spec: + groups: + # The chart ships its own `kopiur.rules` group (monitoring.prometheusRule.enabled + # in the HelmRelease); group and alert names here must not collide with it. + - name: kopiur.freshness.rules + rules: + # Backups were down 44h in July before anyone noticed: nothing watched + # freshness. Snapshots run hourly, so 6h is 6 missed runs, not a slow one. + # Named Overdue, not Stale: the chart's KopiurBackupStale fires off the same + # metric with the same severity, so a shared name is one alert fingerprint. + - alert: KopiurBackupOverdue + expr: |- + time() - kopiur_policy_last_backup_success_timestamp_seconds > 6 * 60 * 60 + for: 30m + annotations: + summary: >- + kopiur {{ $labels.namespace }}/{{ $labels.policy }} has no successful backup + for {{ $value | humanizeDuration }} — restore point is aging + labels: + severity: warning + + - alert: KopiurBackupCritical + expr: |- + time() - kopiur_policy_last_backup_success_timestamp_seconds > 24 * 60 * 60 + for: 30m + annotations: + summary: >- + kopiur {{ $labels.namespace }}/{{ $labels.policy }} has not backed up + successfully in {{ $value | humanizeDuration }} + labels: + severity: critical + + # The July outage removed the CSI snapshot CRDs, so policies stopped reporting + # rather than reporting failure. Absence is the failure mode to catch. + - alert: KopiurPolicyMetricsAbsent + expr: |- + absent(kopiur_policy_last_backup_success_timestamp_seconds) + for: 30m + annotations: + summary: >- + No kopiur policy backup metrics at all — the controller is down or every + policy stopped reporting; backup state is unknown, not healthy + labels: + severity: critical + + # A repo whose snapshot count only ever grows means retention is not pruning; + # a drop to zero means the repo was emptied under us. + - alert: KopiurRepoEmpty + expr: |- + kopiur_repo_snapshot_count == 0 + for: 1h + annotations: + summary: >- + kopiur repository {{ $labels.name }} reports zero snapshots — the repo is + empty or unreadable + labels: + severity: critical + + - alert: KopiurControllerNotLeader + expr: |- + max(kopiur_leader_is_leader) == 0 + for: 30m + annotations: + summary: >- + No kopiur controller holds the leader lease — nothing is scheduling backups + labels: + severity: critical diff --git a/kubernetes/components/kopiur/backup/snapshotpolicy.yaml b/kubernetes/components/kopiur/backup/snapshotpolicy.yaml index c59ab9ba3a..00fc0ffab3 100644 --- a/kubernetes/components/kopiur/backup/snapshotpolicy.yaml +++ b/kubernetes/components/kopiur/backup/snapshotpolicy.yaml @@ -34,6 +34,16 @@ spec: runAsGroup: ${KOPIUR_PGID:=568} capabilities: add: ${KOPIUR_MOVER_CAPS_ADD:=[]} + # Verification is opt-in and nobody had opted in: snapshots ran hourly for 18d with + # LAST-VERIFIED empty on all 18 policies. A repo that stops being restorable reads + # identical to a healthy one until the restore. Fixed hour + 3h jitter spreads the + # fleet wider than `H 3 * * 0` would (a single hashed minute inside one hour), which + # matters because quick verify walks the whole repo. + verification: + quick: + schedule: + cron: 0 3 * * 0 + jitter: 3h compression: compressor: zstd retention: